diff --git a/Cargo.lock b/Cargo.lock index 80000dd3..b015eb6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2764,7 +2764,6 @@ dependencies = [ "serde", "serde_json", "sha2 0.10.9", - "subtle", "temp-env", "tokio", "tokio-test", diff --git a/Cargo.toml b/Cargo.toml index 00655e1e..424c357d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,7 +82,6 @@ regex = "1.12.3" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.149" sha2 = "0.10.9" -subtle = "2.6" temp-env = "0.3.6" tokio = { version = "1.49", features = ["sync", "macros", "io-util", "rt", "time"] } tokio-test = "0.4" diff --git a/crates/integration-tests/fixtures/configs/viceroy-template.toml b/crates/integration-tests/fixtures/configs/viceroy-template.toml index 06c2a850..68e8bd15 100644 --- a/crates/integration-tests/fixtures/configs/viceroy-template.toml +++ b/crates/integration-tests/fixtures/configs/viceroy-template.toml @@ -7,9 +7,17 @@ [local_server.backends] [local_server.kv_stores] - # These inline placeholders satisfy Viceroy's local KV configuration - # requirements without exercising KV-backed application behavior. - [[local_server.kv_stores.creative_store]] + # These inline placeholders satisfy Viceroy's local KV configuration + # requirements without exercising KV-backed application behavior. + [[local_server.kv_stores.counter_store]] + key = "placeholder" + data = "placeholder" + + [[local_server.kv_stores.opid_store]] + key = "placeholder" + data = "placeholder" + + [[local_server.kv_stores.creative_store]] key = "placeholder" data = "placeholder" diff --git a/crates/trusted-server-adapter-fastly/tests/kv_store/counter_store.json b/crates/trusted-server-adapter-fastly/tests/kv_store/counter_store.json new file mode 100644 index 00000000..8c3b91f8 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/tests/kv_store/counter_store.json @@ -0,0 +1,3 @@ +{ + "ec_counter": 10 +} diff --git a/crates/trusted-server-adapter-fastly/tests/kv_store/opid_store.json b/crates/trusted-server-adapter-fastly/tests/kv_store/opid_store.json new file mode 100644 index 00000000..8c3b91f8 --- /dev/null +++ b/crates/trusted-server-adapter-fastly/tests/kv_store/opid_store.json @@ -0,0 +1,3 @@ +{ + "ec_counter": 10 +} diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index bdbb2b01..5ba2d814 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -40,7 +40,6 @@ regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = { workspace = true } -subtle = { workspace = true } tokio = { workspace = true } toml = { workspace = true } trusted-server-js = { path = "../js" } diff --git a/crates/trusted-server-core/src/auth.rs b/crates/trusted-server-core/src/auth.rs index e23fd06d..816891f6 100644 --- a/crates/trusted-server-core/src/auth.rs +++ b/crates/trusted-server-core/src/auth.rs @@ -1,8 +1,6 @@ use base64::{engine::general_purpose::STANDARD, Engine as _}; use fastly::http::{header, StatusCode}; use fastly::{Request, Response}; -use sha2::{Digest as _, Sha256}; -use subtle::ConstantTimeEq as _; use crate::settings::Settings; @@ -32,19 +30,7 @@ pub fn enforce_basic_auth(settings: &Settings, req: &Request) -> Option return Some(unauthorized_response()), }; - // Hash before comparing to normalise lengths — `ct_eq` on raw byte slices - // short-circuits when lengths differ, which would leak credential length. - // SHA-256 produces fixed-size digests so the comparison is truly constant-time. - // - // Note: constant-time guarantees are best-effort on WASM targets because the - // runtime optimiser/JIT may re-introduce variable-time paths. This is an - // inherent limitation of all constant-time code in managed runtimes. - let username_match = Sha256::digest(handler.username.expose().as_bytes()) - .ct_eq(&Sha256::digest(username.as_bytes())); - let password_match = Sha256::digest(handler.password.expose().as_bytes()) - .ct_eq(&Sha256::digest(password.as_bytes())); - - if bool::from(username_match & password_match) { + if handler.username == username && handler.password == password { None } else { Some(unauthorized_response()) @@ -90,11 +76,16 @@ mod tests { use base64::engine::general_purpose::STANDARD; use fastly::http::{header, Method}; - use crate::test_support::tests::create_test_settings; + use crate::test_support::tests::crate_test_settings_str; + + fn settings_with_handlers() -> Settings { + let config = crate_test_settings_str(); + Settings::from_toml(&config).expect("should parse settings with handlers") + } #[test] fn no_challenge_for_non_protected_path() { - let settings = create_test_settings(); + let settings = settings_with_handlers(); let req = Request::new(Method::GET, "https://example.com/open"); assert!(enforce_basic_auth(&settings, &req).is_none()); @@ -102,7 +93,7 @@ mod tests { #[test] fn challenge_when_missing_credentials() { - let settings = create_test_settings(); + let settings = settings_with_handlers(); let req = Request::new(Method::GET, "https://example.com/secure"); let response = enforce_basic_auth(&settings, &req).expect("should challenge"); @@ -115,7 +106,7 @@ mod tests { #[test] fn allow_when_credentials_match() { - let settings = create_test_settings(); + let settings = settings_with_handlers(); let mut req = Request::new(Method::GET, "https://example.com/secure/data"); let token = STANDARD.encode("user:pass"); req.set_header(header::AUTHORIZATION, format!("Basic {token}")); @@ -124,20 +115,19 @@ mod tests { } #[test] - fn challenge_when_both_credentials_wrong() { - let settings = create_test_settings(); + fn challenge_when_credentials_mismatch() { + let settings = settings_with_handlers(); let mut req = Request::new(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("wrong:wrong"); + let token = STANDARD.encode("user:wrong"); req.set_header(header::AUTHORIZATION, format!("Basic {token}")); - let response = enforce_basic_auth(&settings, &req) - .expect("should challenge when both username and password are wrong"); + let response = enforce_basic_auth(&settings, &req).expect("should challenge"); assert_eq!(response.get_status(), StatusCode::UNAUTHORIZED); } #[test] fn challenge_when_scheme_is_not_basic() { - let settings = create_test_settings(); + let settings = settings_with_handlers(); let mut req = Request::new(Method::GET, "https://example.com/secure"); req.set_header(header::AUTHORIZATION, "Bearer token"); @@ -147,7 +137,7 @@ mod tests { #[test] fn allow_admin_path_with_valid_credentials() { - let settings = create_test_settings(); + let settings = settings_with_handlers(); let mut req = Request::new(Method::POST, "https://example.com/admin/keys/rotate"); let token = STANDARD.encode("admin:admin-pass"); req.set_header(header::AUTHORIZATION, format!("Basic {token}")); @@ -160,7 +150,7 @@ mod tests { #[test] fn challenge_admin_path_with_wrong_credentials() { - let settings = create_test_settings(); + let settings = settings_with_handlers(); let mut req = Request::new(Method::POST, "https://example.com/admin/keys/rotate"); let token = STANDARD.encode("admin:wrong"); req.set_header(header::AUTHORIZATION, format!("Basic {token}")); @@ -172,35 +162,11 @@ mod tests { #[test] fn challenge_admin_path_with_missing_credentials() { - let settings = create_test_settings(); + let settings = settings_with_handlers(); let req = Request::new(Method::POST, "https://example.com/admin/keys/rotate"); let response = enforce_basic_auth(&settings, &req) .expect("should challenge admin path with missing credentials"); assert_eq!(response.get_status(), StatusCode::UNAUTHORIZED); } - - #[test] - fn challenge_when_username_wrong_password_correct() { - let settings = create_test_settings(); - let mut req = Request::new(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("wrong:pass"); - req.set_header(header::AUTHORIZATION, format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &req) - .expect("should challenge when only username is wrong"); - assert_eq!(response.get_status(), StatusCode::UNAUTHORIZED); - } - - #[test] - fn challenge_when_username_correct_password_wrong() { - let settings = create_test_settings(); - let mut req = Request::new(Method::GET, "https://example.com/secure/data"); - let token = STANDARD.encode("user:wrong"); - req.set_header(header::AUTHORIZATION, format!("Basic {token}")); - - let response = enforce_basic_auth(&settings, &req) - .expect("should challenge when only password is wrong"); - assert_eq!(response.get_status(), StatusCode::UNAUTHORIZED); - } } diff --git a/crates/trusted-server-core/src/cookies.rs b/crates/trusted-server-core/src/cookies.rs index 544b3801..3408eb79 100644 --- a/crates/trusted-server-core/src/cookies.rs +++ b/crates/trusted-server-core/src/cookies.rs @@ -1,20 +1,15 @@ //! Cookie handling utilities. //! -//! This module provides functionality for parsing and creating cookies +//! This module provides functionality for parsing, stripping, and forwarding cookies //! used in the trusted server system. -use std::borrow::Cow; - use cookie::{Cookie, CookieJar}; use error_stack::{Report, ResultExt}; use fastly::http::header; use fastly::Request; -use crate::constants::{ - COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_TS_EC, COOKIE_US_PRIVACY, -}; +use crate::constants::{COOKIE_EUCONSENT_V2, COOKIE_GPP, COOKIE_GPP_SID, COOKIE_US_PRIVACY}; use crate::error::TrustedServerError; -use crate::settings::Settings; /// Cookie names carrying privacy consent signals. /// @@ -28,44 +23,6 @@ pub const CONSENT_COOKIE_NAMES: &[&str] = &[ COOKIE_US_PRIVACY, ]; -const COOKIE_MAX_AGE: i32 = 365 * 24 * 60 * 60; // 1 year - -fn is_allowed_ec_id_char(c: char) -> bool { - c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_') -} - -#[must_use] -pub(crate) fn ec_id_has_only_allowed_chars(ec_id: &str) -> bool { - ec_id.chars().all(is_allowed_ec_id_char) -} - -fn sanitize_ec_id_for_cookie(ec_id: &str) -> Cow<'_, str> { - if ec_id_has_only_allowed_chars(ec_id) { - return Cow::Borrowed(ec_id); - } - - let safe_id = ec_id - .chars() - .filter(|c| is_allowed_ec_id_char(*c)) - .collect::(); - - log::warn!( - "Stripped disallowed characters from EC ID before setting cookie (len {} -> {}); \ - callers should reject invalid request IDs before cookie creation", - ec_id.len(), - safe_id.len(), - ); - - Cow::Owned(safe_id) -} - -fn ec_cookie_attributes(settings: &Settings, max_age: i32) -> String { - format!( - "Domain={}; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age={max_age}", - settings.publisher.cookie_domain, - ) -} - /// Parses a cookie string into a [`CookieJar`]. /// /// Returns an empty jar if the cookie string is unparseable. @@ -164,106 +121,8 @@ pub fn forward_cookie_header(from: &Request, to: &mut Request, strip_consent: bo } } -/// Returns `true` if every byte in `value` is a valid RFC 6265 `cookie-octet`. -/// An empty string is always rejected. -/// -/// RFC 6265 restricts cookie values to printable US-ASCII excluding whitespace, -/// double-quote, comma, semicolon, and backslash. Rejecting these characters -/// prevents header-injection attacks where a crafted value could append -/// spurious cookie attributes (e.g. `evil; Domain=.attacker.com`). -/// -/// Non-ASCII characters (multi-byte UTF-8) are always rejected because their -/// byte values exceed `0x7E`. -#[must_use] -fn is_safe_cookie_value(value: &str) -> bool { - // RFC 6265 §4.1.1 cookie-octet: - // 0x21 — '!' - // 0x23–0x2B — '#' through '+' (excludes 0x22 DQUOTE) - // 0x2D–0x3A — '-' through ':' (excludes 0x2C comma) - // 0x3C–0x5B — '<' through '[' (excludes 0x3B semicolon) - // 0x5D–0x7E — ']' through '~' (excludes 0x5C backslash, 0x7F DEL) - // All control characters (0x00–0x20) and non-ASCII (0x80+) are also excluded. - !value.is_empty() - && value - .bytes() - .all(|b| matches!(b, 0x21 | 0x23..=0x2B | 0x2D..=0x3A | 0x3C..=0x5B | 0x5D..=0x7E)) -} - -/// Generates a `Set-Cookie` header value with the following security attributes: -/// - `Secure`: transmitted over HTTPS only. -/// - `HttpOnly`: inaccessible to JavaScript (`document.cookie`), blocking XSS exfiltration. -/// Safe to set because integrations receive the EC ID via the `x-ts-ec` -/// response header instead of reading it from the cookie directly. -/// - `SameSite=Lax`: sent on same-site requests and top-level cross-site navigations. -/// `Strict` is intentionally avoided — it would suppress the cookie on the first -/// request when a user arrives from an external page, breaking first-visit attribution. -/// - `Max-Age`: 1 year retention. -/// -/// The `ec_id` is sanitized via an allowlist before embedding in the cookie value. -/// Only ASCII alphanumeric characters and `.`, `-`, `_` are permitted — matching the -/// known EC ID format (`{64-char-hex}.{6-char-alphanumeric}`). Request-sourced IDs -/// with disallowed characters are rejected earlier in [`crate::edge_cookie::get_ec_id`]; -/// this sanitization remains as a defense-in-depth backstop for unexpected callers. -/// -/// The `cookie_domain` is validated at config load time via [`validator::Validate`] on -/// [`crate::settings::Publisher`]; bad config fails at startup, not per-request. -/// -/// # Examples -/// -/// ```no_run -/// # use trusted_server_core::cookies::create_ec_cookie; -/// # use trusted_server_core::settings::Settings; -/// // `settings` is loaded at startup via `Settings::from_toml_and_env`. -/// # fn example(settings: &Settings) { -/// let cookie = create_ec_cookie(settings, "abc123.xk92ab"); -/// assert!(cookie.contains("HttpOnly")); -/// assert!(cookie.contains("Secure")); -/// # } -/// ``` -#[must_use] -pub fn create_ec_cookie(settings: &Settings, ec_id: &str) -> String { - let safe_id = sanitize_ec_id_for_cookie(ec_id); - - format!( - "{}={}; {}", - COOKIE_TS_EC, - safe_id, - ec_cookie_attributes(settings, COOKIE_MAX_AGE), - ) -} - -/// Sets the EC ID cookie on the given response. -/// -/// Validates `ec_id` against RFC 6265 `cookie-octet` rules before -/// interpolation. If the value contains unsafe characters (e.g. semicolons), -/// the cookie is not set and a warning is logged. This prevents an attacker -/// from injecting spurious cookie attributes via a controlled ID value. -/// -/// `cookie_domain` comes from operator configuration and is considered trusted. -pub fn set_ec_cookie(settings: &Settings, response: &mut fastly::Response, ec_id: &str) { - if !is_safe_cookie_value(ec_id) { - log::warn!( - "Rejecting EC ID for Set-Cookie: value of {} bytes contains characters illegal in a cookie value", - ec_id.len() - ); - return; - } - response.append_header(header::SET_COOKIE, create_ec_cookie(settings, ec_id)); -} - -/// Expires the EC cookie by setting `Max-Age=0`. -/// -/// Used when a user revokes consent — the browser will delete the cookie -/// on receipt of this header. -pub fn expire_ec_cookie(settings: &Settings, response: &mut fastly::Response) { - let cookie = format!("{}=; {}", COOKIE_TS_EC, ec_cookie_attributes(settings, 0),); - response.append_header(header::SET_COOKIE, cookie); -} - #[cfg(test)] mod tests { - use crate::test_support::tests::create_test_settings; - use super::*; #[test] @@ -286,7 +145,7 @@ mod tests { } #[test] - fn test_parse_cookies_to_jar_empty() { + fn test_parse_cookies_to_jar_emtpy() { let cookie_str = ""; let jar = parse_cookies_to_jar(cookie_str); @@ -341,161 +200,6 @@ mod tests { assert!(jar.iter().count() == 0); } - #[test] - fn test_set_ec_cookie() { - let settings = create_test_settings(); - let mut response = fastly::Response::new(); - set_ec_cookie(&settings, &mut response, "abc123.XyZ789"); - - let cookie_str = response - .get_header(header::SET_COOKIE) - .expect("Set-Cookie header should be present") - .to_str() - .expect("header should be valid UTF-8"); - - assert_eq!( - cookie_str, - format!( - "{}=abc123.XyZ789; Domain={}; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age={}", - COOKIE_TS_EC, settings.publisher.cookie_domain, COOKIE_MAX_AGE, - ), - "Set-Cookie header should match expected format" - ); - } - - #[test] - fn test_create_ec_cookie_sanitizes_disallowed_chars_in_id() { - let settings = create_test_settings(); - // Allowlist permits only ASCII alphanumeric, '.', '-', '_'. - // ';', '=', '\r', '\n', spaces, NUL bytes, and other control chars are all stripped. - let result = create_ec_cookie(&settings, "evil;injected\r\nfoo=bar\0baz"); - // Extract the value portion anchored to the cookie name constant to - // avoid false positives from disallowed chars in cookie attributes. - let value = result - .strip_prefix(&format!("{}=", COOKIE_TS_EC)) - .and_then(|s| s.split_once(';').map(|(v, _)| v)) - .expect("should have cookie value portion"); - assert_eq!( - value, "evilinjectedfoobarbaz", - "should strip disallowed characters and preserve safe chars" - ); - } - - #[test] - fn test_create_ec_cookie_preserves_well_formed_id() { - let settings = create_test_settings(); - // A well-formed ID should pass through the allowlist unmodified. - let id = "abc123def0123456789abcdef0123456789abcdef0123456789abcdef01234567.xk92ab"; - let result = create_ec_cookie(&settings, id); - let value = result - .strip_prefix(&format!("{}=", COOKIE_TS_EC)) - .and_then(|s| s.split_once(';').map(|(v, _)| v)) - .expect("should have cookie value portion"); - assert_eq!(value, id, "should not modify a well-formed EC ID"); - } - - #[test] - fn test_set_ec_cookie_rejects_semicolon() { - let settings = create_test_settings(); - let mut response = fastly::Response::new(); - set_ec_cookie(&settings, &mut response, "evil; Domain=.attacker.com"); - - assert!( - response.get_header(header::SET_COOKIE).is_none(), - "Set-Cookie should not be set when value contains a semicolon" - ); - } - - #[test] - fn test_set_ec_cookie_rejects_crlf() { - let settings = create_test_settings(); - let mut response = fastly::Response::new(); - set_ec_cookie(&settings, &mut response, "evil\r\nX-Injected: header"); - - assert!( - response.get_header(header::SET_COOKIE).is_none(), - "Set-Cookie should not be set when value contains CRLF" - ); - } - - #[test] - fn test_set_ec_cookie_rejects_space() { - let settings = create_test_settings(); - let mut response = fastly::Response::new(); - set_ec_cookie(&settings, &mut response, "bad value"); - - assert!( - response.get_header(header::SET_COOKIE).is_none(), - "Set-Cookie should not be set when value contains whitespace" - ); - } - - #[test] - fn test_is_safe_cookie_value_rejects_empty_string() { - assert!(!is_safe_cookie_value(""), "should reject empty string"); - } - - #[test] - fn test_is_safe_cookie_value_accepts_valid_ec_id_characters() { - // Hex digits, dot separator, alphanumeric suffix — the full EC ID character set - assert!( - is_safe_cookie_value("abcdef0123456789.ABCDEFabcdef"), - "should accept hex digits, dots, and alphanumeric characters" - ); - } - - #[test] - fn test_is_safe_cookie_value_rejects_non_ascii() { - assert!( - !is_safe_cookie_value("valüe"), - "should reject non-ASCII UTF-8 characters" - ); - } - - #[test] - fn test_is_safe_cookie_value_rejects_illegal_characters() { - assert!(!is_safe_cookie_value("val;ue"), "should reject semicolon"); - assert!(!is_safe_cookie_value("val,ue"), "should reject comma"); - assert!( - !is_safe_cookie_value("val\"ue"), - "should reject double-quote" - ); - assert!(!is_safe_cookie_value("val\\ue"), "should reject backslash"); - assert!(!is_safe_cookie_value("val ue"), "should reject space"); - assert!( - !is_safe_cookie_value("val\x00ue"), - "should reject null byte" - ); - assert!( - !is_safe_cookie_value("val\x7fue"), - "should reject DEL character" - ); - } - - #[test] - fn test_expire_ec_cookie_matches_security_attributes() { - let settings = create_test_settings(); - let mut response = fastly::Response::new(); - - expire_ec_cookie(&settings, &mut response); - - let cookie_header = response - .get_header(header::SET_COOKIE) - .expect("Set-Cookie header should be present"); - let cookie_str = cookie_header - .to_str() - .expect("header should be valid UTF-8"); - - assert_eq!( - cookie_str, - format!( - "{}=; Domain={}; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age=0", - COOKIE_TS_EC, settings.publisher.cookie_domain, - ), - "expiry cookie should retain the same security attributes as the live cookie" - ); - } - // --------------------------------------------------------------- // strip_cookies tests // --------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/ec/kv.rs b/crates/trusted-server-core/src/ec/kv.rs new file mode 100644 index 00000000..44be5513 --- /dev/null +++ b/crates/trusted-server-core/src/ec/kv.rs @@ -0,0 +1,588 @@ +//! KV identity graph operations. +//! +//! This module provides [`KvIdentityGraph`] which wraps a Fastly KV Store +//! and implements the read-modify-write operations for the EC identity graph. +//! +//! All methods return `Result` — callers decide whether to swallow errors +//! (organic request paths) or propagate them (sync endpoints). See the +//! per-operation error handling policy in the spec §7.5. + +use std::time::Duration; + +use error_stack::{Report, ResultExt}; +use fastly::kv_store::{InsertMode, KVStore}; + +use crate::error::TrustedServerError; + +use super::kv_types::{KvEntry, KvMetadata}; + +/// Maximum number of CAS retry attempts before giving up. +const MAX_CAS_RETRIES: u32 = 3; + +/// Minimum interval (seconds) between `last_seen` KV writes for the same key. +/// Prevents write thrashing under bursty traffic — Fastly KV enforces a +/// 1 write/sec limit per key. +const LAST_SEEN_DEBOUNCE_SECS: u64 = 300; + +/// TTL for live entries (1 year), matching the EC cookie `Max-Age`. +const ENTRY_TTL: Duration = Duration::from_secs(365 * 24 * 60 * 60); + +/// TTL for withdrawal tombstones (24 hours). +const TOMBSTONE_TTL: Duration = Duration::from_secs(24 * 60 * 60); + +/// Wraps a Fastly KV Store for EC identity graph operations. +/// +/// Each EC hash (64-char hex prefix) maps to a JSON-encoded [`KvEntry`] +/// containing consent state, geo location, and accumulated partner IDs. +/// +/// Methods use optimistic concurrency (generation markers) for safe +/// read-modify-write operations on concurrent requests. +pub struct KvIdentityGraph { + store_name: String, +} + +impl KvIdentityGraph { + /// Creates a new identity graph backed by the named KV store. + #[must_use] + pub fn new(store_name: impl Into) -> Self { + Self { + store_name: store_name.into(), + } + } + + /// Returns the configured store name. + #[must_use] + pub fn store_name(&self) -> &str { + &self.store_name + } + + /// Opens the underlying Fastly KV store. + fn open_store(&self) -> Result> { + KVStore::open(&self.store_name) + .change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: "Failed to open KV store".to_owned(), + })? + .ok_or_else(|| { + Report::new(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: "KV store not found".to_owned(), + }) + }) + } + + /// Serializes an entry body and metadata for insertion. + fn serialize_entry( + entry: &KvEntry, + store_name: &str, + ) -> Result<(String, String), Report> { + let body = serde_json::to_string(entry).change_context(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: "Failed to serialize KV entry body".to_owned(), + })?; + let meta = KvMetadata::from_entry(entry); + let meta_str = + serde_json::to_string(&meta).change_context(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: "Failed to serialize KV entry metadata".to_owned(), + })?; + Ok((body, meta_str)) + } + + /// Reads the full entry and its generation marker for CAS writes. + /// + /// Returns `Ok(None)` when the key does not exist. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store open or read failure. + pub fn get(&self, ec_hash: &str) -> Result, Report> { + let store = self.open_store()?; + let mut response = match store.lookup(ec_hash) { + Ok(resp) => resp, + Err(fastly::kv_store::KVStoreError::ItemNotFound) => return Ok(None), + Err(err) => { + return Err( + Report::new(err).change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Failed to read key '{ec_hash}'"), + }), + ); + } + }; + + let generation = response.current_generation(); + let body_bytes = response.take_body_bytes(); + let entry: KvEntry = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Failed to deserialize entry for key '{ec_hash}'"), + })?; + + Ok(Some((entry, generation))) + } + + /// Reads only the metadata for an EC hash (no body streaming). + /// + /// Returns `Ok(None)` when the key does not exist or has no metadata. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store open or read failure. + pub fn get_metadata( + &self, + ec_hash: &str, + ) -> Result, Report> { + let store = self.open_store()?; + let response = match store.lookup(ec_hash) { + Ok(resp) => resp, + Err(fastly::kv_store::KVStoreError::ItemNotFound) => return Ok(None), + Err(err) => { + return Err( + Report::new(err).change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Failed to read metadata for key '{ec_hash}'"), + }), + ); + } + }; + + let meta_bytes = match response.metadata() { + Some(bytes) => bytes, + None => return Ok(None), + }; + + let meta: KvMetadata = + serde_json::from_slice(&meta_bytes).change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Failed to deserialize metadata for key '{ec_hash}'"), + })?; + + Ok(Some(meta)) + } + + /// Creates a new entry. Fails if the key already exists. + /// + /// Uses `InsertMode::Add` so concurrent creates for the same EC hash + /// are safely rejected (only one wins). + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store error or if the + /// key already exists (`ItemPreconditionFailed`). + pub fn create(&self, ec_hash: &str, entry: &KvEntry) -> Result<(), Report> { + let store = self.open_store()?; + let (body, meta_str) = Self::serialize_entry(entry, &self.store_name)?; + let created = Self::try_insert_add(&store, ec_hash, &body, &meta_str, &self.store_name)?; + if created { + Ok(()) + } else { + Err(Report::new(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Key '{ec_hash}' already exists"), + })) + } + } + + /// Low-level create using a pre-opened store and pre-serialized data. + /// + /// Returns `true` if the entry was created, `false` if the key already + /// exists (`ItemPreconditionFailed`). Other errors are propagated. + fn try_insert_add( + store: &KVStore, + ec_hash: &str, + body: &str, + meta_str: &str, + store_name: &str, + ) -> Result> { + match store + .build_insert() + .mode(InsertMode::Add) + .metadata(meta_str) + .time_to_live(ENTRY_TTL) + .execute(ec_hash, body) + { + Ok(()) => Ok(true), + Err(fastly::kv_store::KVStoreError::ItemPreconditionFailed) => Ok(false), + Err(err) => Err( + Report::new(err).change_context(TrustedServerError::KvStore { + store_name: store_name.to_owned(), + message: format!("Failed to create entry for key '{ec_hash}'"), + }), + ), + } + } + + /// Creates a new entry, or overwrites an existing tombstone on re-consent. + /// + /// Three-way behavior: + /// - **No existing key** — creates the entry (same as [`create`](Self::create)). + /// - **Existing live entry** (`consent.ok = true`) — no-op, returns `Ok(())`. + /// - **Existing tombstone** (`consent.ok = false`) — CAS overwrite with + /// the new entry. Retries up to [`MAX_CAS_RETRIES`] on conflict. + /// + /// Called by `generate_if_needed()` instead of `create()` so that a + /// user who re-consents within the 24-hour tombstone window recovers + /// immediately. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store error or CAS + /// exhaustion. + pub fn create_or_revive( + &self, + ec_hash: &str, + entry: &KvEntry, + ) -> Result<(), Report> { + // Serialize once and reuse across the fast path and CAS loop. + let store = self.open_store()?; + let (body, meta_str) = Self::serialize_entry(entry, &self.store_name)?; + + // Try create first — fast path for new entries. + if Self::try_insert_add(&store, ec_hash, &body, &meta_str, &self.store_name)? { + return Ok(()); + } + + // Key exists — read it to determine if it's live or a tombstone. + let (existing, generation) = match self.get(ec_hash)? { + Some(pair) => pair, + // Raced with a delete — try create again. + None => return self.create(ec_hash, entry), + }; + + // Live entry — nothing to do. + if existing.consent.ok { + log::debug!("create_or_revive: live entry exists for '{ec_hash}', no-op"); + return Ok(()); + } + + // Tombstone — CAS overwrite to revive. + log::info!("create_or_revive: reviving tombstone for '{ec_hash}'"); + + let mut current_gen = generation; + for attempt in 0..MAX_CAS_RETRIES { + match store + .build_insert() + .if_generation_match(current_gen) + .metadata(&meta_str) + .time_to_live(ENTRY_TTL) + .execute(ec_hash, body.as_str()) + { + Ok(()) => return Ok(()), + Err(fastly::kv_store::KVStoreError::ItemPreconditionFailed) => { + log::debug!( + "create_or_revive: CAS conflict on attempt {}/{MAX_CAS_RETRIES} for '{ec_hash}'", + attempt + 1, + ); + // Re-read to get fresh generation. + match self.get(ec_hash)? { + Some((refreshed, gen)) => { + if refreshed.consent.ok { + // Someone else revived it — done. + return Ok(()); + } + current_gen = gen; + } + None => return self.create(ec_hash, entry), + } + } + Err(err) => { + return Err( + Report::new(err).change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!( + "Failed to revive tombstone for key '{ec_hash}' on attempt {}", + attempt + 1, + ), + }), + ); + } + } + } + + Err(Report::new(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!( + "CAS conflict after {MAX_CAS_RETRIES} retries reviving tombstone for '{ec_hash}'" + ), + })) + } + + /// Atomically merges a partner ID into the existing entry. + /// + /// Uses CAS (generation markers) to avoid clobbering concurrent writes + /// from other partners. Retries up to [`MAX_CAS_RETRIES`] on conflict. + /// + /// If the root entry does not exist (e.g. the initial `create_or_revive` + /// failed), creates a minimal live entry first — this is the recovery + /// path for best-effort EC creation misses. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store error or CAS + /// exhaustion after [`MAX_CAS_RETRIES`] attempts. + pub fn upsert_partner_id( + &self, + ec_hash: &str, + partner_id: &str, + uid: &str, + synced: u64, + ) -> Result<(), Report> { + // Open store once for write operations. Note: `self.get()` opens + // its own handle internally — this is intentional since `KVStore::open` + // is a cheap name lookup, and keeping the read/write APIs independent + // simplifies the method signatures. + let store = self.open_store()?; + + for attempt in 0..MAX_CAS_RETRIES { + let (mut entry, generation) = match self.get(ec_hash)? { + Some(pair) => pair, + None => { + // Root entry missing — create a minimal entry. + log::info!( + "upsert_partner_id: no entry for '{ec_hash}', creating minimal entry" + ); + let minimal = KvEntry::minimal(partner_id, uid, synced); + let (min_body, min_meta) = Self::serialize_entry(&minimal, &self.store_name)?; + if Self::try_insert_add( + &store, + ec_hash, + &min_body, + &min_meta, + &self.store_name, + )? { + return Ok(()); + } + // Key appeared between get() and create — re-read on next iteration. + log::debug!( + "upsert_partner_id: minimal create raced for '{ec_hash}', retrying (attempt {}/{})", + attempt + 1, + MAX_CAS_RETRIES, + ); + continue; + } + }; + + // Reject upserts on withdrawn entries — a late sync must not + // repopulate partner IDs after consent withdrawal. + if !entry.consent.ok { + log::info!( + "upsert_partner_id: entry for '{ec_hash}' is a tombstone, rejecting upsert" + ); + return Err(Report::new(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!( + "Cannot upsert partner '{partner_id}' for withdrawn key '{ec_hash}'" + ), + })); + } + + // Merge the partner ID. + entry.ids.insert( + partner_id.to_owned(), + super::kv_types::KvPartnerId { + uid: uid.to_owned(), + synced, + }, + ); + + let (body, meta_str) = Self::serialize_entry(&entry, &self.store_name)?; + + match store + .build_insert() + .if_generation_match(generation) + .metadata(&meta_str) + .time_to_live(ENTRY_TTL) + .execute(ec_hash, body.as_str()) + { + Ok(()) => return Ok(()), + Err(fastly::kv_store::KVStoreError::ItemPreconditionFailed) => { + log::debug!( + "upsert_partner_id: CAS conflict on attempt {}/{MAX_CAS_RETRIES} for '{ec_hash}'", + attempt + 1, + ); + // Loop will re-read on next iteration. + } + Err(err) => { + return Err( + Report::new(err).change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!( + "Failed to upsert partner '{partner_id}' for key '{ec_hash}'" + ), + }), + ); + } + } + } + + Err(Report::new(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!( + "CAS conflict after {MAX_CAS_RETRIES} retries upserting partner '{partner_id}' for '{ec_hash}'" + ), + })) + } + + /// Updates the `last_seen` timestamp with a 300-second debounce. + /// + /// Skips the write if the stored `last_seen` is within + /// [`LAST_SEEN_DEBOUNCE_SECS`] of the new timestamp, or if the entry + /// does not exist. + /// + /// Does **not** retry on CAS conflict — if someone else wrote more + /// recently, the debounce condition is satisfied anyway. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store error or CAS + /// conflict. + pub fn update_last_seen( + &self, + ec_hash: &str, + timestamp: u64, + ) -> Result<(), Report> { + let (mut entry, generation) = match self.get(ec_hash)? { + Some(pair) => pair, + None => { + log::debug!("update_last_seen: no entry for '{ec_hash}', skipping"); + return Ok(()); + } + }; + + // Skip tombstones — a stale cookie should not extend a 24h tombstone + // back to 1-year TTL. + if !entry.consent.ok { + log::debug!("update_last_seen: entry for '{ec_hash}' is a tombstone, skipping"); + return Ok(()); + } + + // Guard against stale/out-of-order timestamps. + if timestamp <= entry.last_seen { + log::trace!( + "update_last_seen: stale timestamp for '{ec_hash}' (stored={}, incoming={timestamp})", + entry.last_seen, + ); + return Ok(()); + } + + // Debounce: skip if the stored value is recent enough. + if timestamp - entry.last_seen < LAST_SEEN_DEBOUNCE_SECS { + log::trace!( + "update_last_seen: debounced for '{ec_hash}' (delta={}s)", + timestamp - entry.last_seen, + ); + return Ok(()); + } + + entry.last_seen = timestamp; + let store = self.open_store()?; + let (body, meta_str) = Self::serialize_entry(&entry, &self.store_name)?; + + store + .build_insert() + .if_generation_match(generation) + .metadata(&meta_str) + .time_to_live(ENTRY_TTL) + .execute(ec_hash, body) + .change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Failed to update last_seen for key '{ec_hash}'"), + }) + } + + /// Writes a withdrawal tombstone for consent enforcement. + /// + /// Overwrites the entry with `consent.ok = false`, empty partner IDs, + /// and a 24-hour TTL. Uses unconditional overwrite (no CAS) since the + /// entry is being withdrawn regardless of concurrent state. + /// + /// The tombstone allows batch sync clients (`POST /api/v1/sync`) to + /// distinguish `consent_withdrawn` from `ec_hash_not_found` for 24 hours. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store error. Callers on + /// the browser path should log at `error` level and continue — cookie + /// deletion is the primary enforcement mechanism. + pub fn write_withdrawal_tombstone( + &self, + ec_hash: &str, + ) -> Result<(), Report> { + let store = self.open_store()?; + let entry = KvEntry::tombstone(current_timestamp()); + let (body, meta_str) = Self::serialize_entry(&entry, &self.store_name)?; + + store + .build_insert() + .metadata(&meta_str) + .time_to_live(TOMBSTONE_TTL) + .execute(ec_hash, body) + .change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Failed to write tombstone for key '{ec_hash}'"), + }) + } + + /// Hard-deletes the entry. + /// + /// Reserved for the IAB data deletion framework (deferred). For consent + /// withdrawal, use [`write_withdrawal_tombstone`](Self::write_withdrawal_tombstone). + /// + /// # Errors + /// + /// Returns [`TrustedServerError::KvStore`] on store error. + pub fn delete(&self, ec_hash: &str) -> Result<(), Report> { + let store = self.open_store()?; + store + .delete(ec_hash) + .change_context(TrustedServerError::KvStore { + store_name: self.store_name.clone(), + message: format!("Failed to delete key '{ec_hash}'"), + }) + } +} + +/// Returns the current Unix timestamp in seconds. +/// +/// Uses `std::time::SystemTime` which is supported on `wasm32-wasip1`. +fn current_timestamp() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn constants_have_expected_values() { + assert_eq!(MAX_CAS_RETRIES, 3); + assert_eq!(LAST_SEEN_DEBOUNCE_SECS, 300); + assert_eq!(ENTRY_TTL, Duration::from_secs(31_536_000)); + assert_eq!(TOMBSTONE_TTL, Duration::from_secs(86_400)); + } + + #[test] + fn current_timestamp_is_nonzero() { + let ts = current_timestamp(); + assert!(ts > 0, "should return a nonzero timestamp"); + } + + #[test] + fn serialize_entry_produces_valid_json() { + let entry = KvEntry::tombstone(1000); + let (body, meta) = + KvIdentityGraph::serialize_entry(&entry, "test-store").expect("should serialize entry"); + + // Verify body is valid JSON. + let _: KvEntry = + serde_json::from_str(&body).expect("should deserialize body back to KvEntry"); + + // Verify metadata is valid JSON. + let _: KvMetadata = + serde_json::from_str(&meta).expect("should deserialize metadata back to KvMetadata"); + } +} diff --git a/crates/trusted-server-core/src/ec/kv_types.rs b/crates/trusted-server-core/src/ec/kv_types.rs new file mode 100644 index 00000000..fa0f256f --- /dev/null +++ b/crates/trusted-server-core/src/ec/kv_types.rs @@ -0,0 +1,386 @@ +//! KV identity graph schema types. +//! +//! These types define the JSON schema stored in the Fastly KV Store for the +//! EC identity graph. Each EC hash (64-char hex prefix) maps to a [`KvEntry`] +//! containing consent state, geo location, and accumulated partner IDs. +//! +//! The schema is versioned (`v: 1`) to allow future migrations. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::consent::ConsentContext; +use crate::geo::GeoInfo; + +/// Current schema version for KV entries. +pub const SCHEMA_VERSION: u8 = 1; + +/// Full KV entry stored as the body of an EC identity graph record. +/// +/// **KV key:** 64-character hex hash (the stable prefix from the EC ID). +/// **KV value:** JSON-serialized `KvEntry` (max ~5KB). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvEntry { + /// Schema version — always [`SCHEMA_VERSION`]. + pub v: u8, + /// Unix timestamp (seconds) of initial entry creation. + pub created: u64, + /// Unix timestamp (seconds) of last organic request. + /// Updated by [`super::kv::KvIdentityGraph::update_last_seen`] with + /// a 300-second debounce. + pub last_seen: u64, + /// Consent state sub-object. + pub consent: KvConsent, + /// Geo location sub-object. + pub geo: KvGeo, + /// Map of partner ID namespace → synced UID record. + /// Populated by pixel sync, batch sync, and pull sync operations. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub ids: HashMap, +} + +/// Consent state within a KV entry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvConsent { + /// Raw TCF v2 consent string. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tcf: Option, + /// Raw GPP consent string. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub gpp: Option, + /// `true` for a live entry, `false` for a withdrawal tombstone. + pub ok: bool, + /// Unix timestamp (seconds) of last consent state change. + pub updated: u64, +} + +/// Geo location within a KV entry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvGeo { + /// ISO 3166-1 alpha-2 country code (e.g. `"US"`). + pub country: String, + /// ISO 3166-2 region code (e.g. `"CA"` for California). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub region: Option, +} + +/// A synced partner user ID within a KV entry. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvPartnerId { + /// The partner's user identifier. + pub uid: String, + /// Unix timestamp (seconds) when this UID was written/updated. + pub synced: u64, +} + +/// Compact metadata stored alongside the KV entry body. +/// +/// Fastly KV metadata is limited to 2048 bytes and can be read without +/// streaming the full body. Used by batch sync for fast consent checks. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct KvMetadata { + /// Mirrors [`KvConsent::ok`] — `false` means tombstone. + pub ok: bool, + /// Mirrors [`KvGeo::country`]. + pub country: String, + /// Mirrors [`KvEntry::v`]. + pub v: u8, +} + +impl KvEntry { + /// Creates a new live entry from the current request context. + #[must_use] + pub fn new(consent: &ConsentContext, geo: Option<&GeoInfo>, now: u64) -> Self { + Self { + v: SCHEMA_VERSION, + created: now, + last_seen: now, + consent: KvConsent { + tcf: consent.raw_tc_string.clone(), + gpp: consent.raw_gpp_string.clone(), + ok: true, + updated: now, + }, + geo: KvGeo::from_geo_info(geo), + ids: HashMap::new(), + } + } + + /// Creates a minimal live entry for the recovery path. + /// + /// Used by [`super::kv::KvIdentityGraph::upsert_partner_id`] when the + /// root KV entry is missing (e.g. the initial best-effort + /// `create_or_revive` failed on EC generation). + #[must_use] + pub fn minimal(partner_id: &str, uid: &str, synced: u64) -> Self { + let mut ids = HashMap::new(); + ids.insert( + partner_id.to_owned(), + KvPartnerId { + uid: uid.to_owned(), + synced, + }, + ); + Self { + v: SCHEMA_VERSION, + created: synced, + last_seen: synced, + consent: KvConsent { + tcf: None, + gpp: None, + ok: true, + updated: synced, + }, + geo: KvGeo { + country: "ZZ".to_owned(), + region: None, + }, + ids, + } + } + + /// Creates a withdrawal tombstone entry. + /// + /// Sets `consent.ok = false`, clears all partner IDs, and uses a + /// placeholder geo. The caller should apply a 24-hour TTL when writing. + /// + /// **Note:** The original `created` timestamp is intentionally not + /// preserved — reading the existing entry first would add latency on + /// the consent-withdrawal hot path, and the tombstone expires in 24h. + #[must_use] + pub fn tombstone(now: u64) -> Self { + Self { + v: SCHEMA_VERSION, + created: now, + last_seen: now, + consent: KvConsent { + tcf: None, + gpp: None, + ok: false, + updated: now, + }, + geo: KvGeo { + country: "ZZ".to_owned(), + region: None, + }, + ids: HashMap::new(), + } + } +} + +impl KvMetadata { + /// Extracts metadata from a full entry. + #[must_use] + pub fn from_entry(entry: &KvEntry) -> Self { + Self { + ok: entry.consent.ok, + country: entry.geo.country.clone(), + v: entry.v, + } + } +} + +impl KvGeo { + /// Creates a `KvGeo` from an optional [`GeoInfo`]. + /// + /// Returns `country: "ZZ"` (unknown) when geo data is unavailable. + #[must_use] + pub fn from_geo_info(geo: Option<&GeoInfo>) -> Self { + match geo { + Some(info) => Self { + country: info.country.clone(), + region: info.region.clone(), + }, + None => Self { + country: "ZZ".to_owned(), + region: None, + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_consent_context() -> ConsentContext { + ConsentContext { + raw_tc_string: Some("CP_test_tc_string".to_owned()), + raw_gpp_string: Some("DBA_test_gpp".to_owned()), + ..ConsentContext::default() + } + } + + fn sample_geo_info() -> GeoInfo { + GeoInfo { + city: "San Francisco".to_owned(), + country: "US".to_owned(), + continent: "NorthAmerica".to_owned(), + latitude: 37.7749, + longitude: -122.4194, + metro_code: 807, + region: Some("CA".to_owned()), + } + } + + #[test] + fn entry_serialization_roundtrip() { + let geo = sample_geo_info(); + let consent = sample_consent_context(); + let mut entry = KvEntry::new(&consent, Some(&geo), 1741824000); + entry.ids.insert( + "liveramp".to_owned(), + KvPartnerId { + uid: "LR_xyz".to_owned(), + synced: 1741890000, + }, + ); + + let json = serde_json::to_string(&entry).expect("should serialize KvEntry"); + let deserialized: KvEntry = + serde_json::from_str(&json).expect("should deserialize KvEntry"); + + assert_eq!(deserialized.v, SCHEMA_VERSION); + assert_eq!(deserialized.created, 1741824000); + assert_eq!( + deserialized.consent.tcf.as_deref(), + Some("CP_test_tc_string") + ); + assert_eq!(deserialized.consent.gpp.as_deref(), Some("DBA_test_gpp")); + assert!(deserialized.consent.ok, "should be a live entry"); + assert_eq!(deserialized.geo.country, "US"); + assert_eq!(deserialized.geo.region.as_deref(), Some("CA")); + assert_eq!( + deserialized.ids.get("liveramp").map(|p| p.uid.as_str()), + Some("LR_xyz"), + ); + } + + #[test] + fn metadata_serialization_roundtrip() { + let meta = KvMetadata { + ok: true, + country: "US".to_owned(), + v: 1, + }; + + let json = serde_json::to_string(&meta).expect("should serialize KvMetadata"); + let deserialized: KvMetadata = + serde_json::from_str(&json).expect("should deserialize KvMetadata"); + + assert!(deserialized.ok, "should be ok=true"); + assert_eq!(deserialized.country, "US"); + assert_eq!(deserialized.v, 1); + } + + #[test] + fn metadata_fits_in_2048_bytes() { + // Worst case: long country code (though ISO 3166-1 is always 2 chars) + let meta = KvMetadata { + ok: false, + country: "XX".to_owned(), + v: SCHEMA_VERSION, + }; + let json = serde_json::to_string(&meta).expect("should serialize KvMetadata"); + assert!( + json.len() <= 2048, + "metadata must fit in Fastly's 2048-byte limit, got {} bytes", + json.len() + ); + } + + #[test] + fn new_entry_has_correct_initial_state() { + let consent = sample_consent_context(); + let geo = sample_geo_info(); + let entry = KvEntry::new(&consent, Some(&geo), 1000); + + assert_eq!(entry.v, SCHEMA_VERSION); + assert_eq!(entry.created, 1000); + assert_eq!(entry.last_seen, 1000); + assert!(entry.consent.ok, "should be a live entry"); + assert_eq!(entry.consent.updated, 1000); + assert_eq!(entry.geo.country, "US"); + assert!(entry.ids.is_empty(), "should have no partner IDs initially"); + } + + #[test] + fn new_entry_without_geo_uses_zz() { + let consent = ConsentContext::default(); + let entry = KvEntry::new(&consent, None, 1000); + assert_eq!( + entry.geo.country, "ZZ", + "should use ZZ when geo is unavailable" + ); + assert!(entry.geo.region.is_none()); + } + + #[test] + fn minimal_entry_has_partner_id_and_placeholder_geo() { + let entry = KvEntry::minimal("ssp_x", "abc123", 1741824000); + + assert_eq!(entry.v, SCHEMA_VERSION); + assert!(entry.consent.ok, "should be a live entry"); + assert_eq!(entry.geo.country, "ZZ"); + assert_eq!(entry.ids.len(), 1); + let partner = entry.ids.get("ssp_x").expect("should have ssp_x entry"); + assert_eq!(partner.uid, "abc123"); + assert_eq!(partner.synced, 1741824000); + } + + #[test] + fn tombstone_entry_has_correct_shape() { + let entry = KvEntry::tombstone(1741910400); + + assert_eq!(entry.v, SCHEMA_VERSION); + assert!(!entry.consent.ok, "should be a tombstone"); + assert!(entry.ids.is_empty(), "tombstone should have no partner IDs"); + assert_eq!(entry.geo.country, "ZZ"); + assert_eq!(entry.consent.updated, 1741910400); + } + + #[test] + fn metadata_from_entry_mirrors_fields() { + let consent = sample_consent_context(); + let geo = sample_geo_info(); + let entry = KvEntry::new(&consent, Some(&geo), 1000); + let meta = KvMetadata::from_entry(&entry); + + assert_eq!(meta.ok, entry.consent.ok); + assert_eq!(meta.country, entry.geo.country); + assert_eq!(meta.v, entry.v); + } + + #[test] + fn tombstone_metadata_has_ok_false() { + let entry = KvEntry::tombstone(1000); + let meta = KvMetadata::from_entry(&entry); + + assert!(!meta.ok, "tombstone metadata should have ok=false"); + } + + #[test] + fn empty_ids_omitted_from_json() { + let entry = KvEntry::tombstone(1000); + let json = serde_json::to_string(&entry).expect("should serialize"); + assert!( + !json.contains("\"ids\""), + "empty ids should be omitted from JSON, got: {json}" + ); + } + + #[test] + fn none_consent_fields_omitted_from_json() { + let entry = KvEntry::tombstone(1000); + let json = serde_json::to_string(&entry).expect("should serialize"); + assert!( + !json.contains("\"tcf\""), + "None tcf should be omitted from JSON" + ); + assert!( + !json.contains("\"gpp\""), + "None gpp should be omitted from JSON" + ); + } +} diff --git a/crates/trusted-server-core/src/ec/mod.rs b/crates/trusted-server-core/src/ec/mod.rs index 6f53b12e..97116239 100644 --- a/crates/trusted-server-core/src/ec/mod.rs +++ b/crates/trusted-server-core/src/ec/mod.rs @@ -16,10 +16,14 @@ //! - [`generation`] — HMAC-based ID generation, IP normalization, format helpers //! - [`consent`] — EC-specific consent gating wrapper //! - [`cookies`] — `Set-Cookie` header creation and expiration helpers +//! - [`kv`] — KV Store identity graph operations (CAS, tombstones, debounce) +//! - [`kv_types`] — Schema types for KV identity graph entries pub mod consent; pub mod cookies; pub mod generation; +pub mod kv; +pub mod kv_types; use cookie::CookieJar; use error_stack::Report; @@ -32,7 +36,7 @@ use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::settings::Settings; -pub use generation::{ec_hash, extract_client_ip, generate_ec_id, is_valid_ec_id}; +pub use generation::{ec_hash, generate_ec_id, is_valid_ec_id}; /// Parsed EC identity from an incoming request. /// @@ -92,36 +96,6 @@ pub fn get_ec_id(req: &fastly::Request) -> Result, Report Result> { - if let Some(id) = get_ec_id(req)? { - return Ok(id); - } - - // Fallback to "unknown" when client IP is unavailable (e.g. local testing). - // All such requests share the same HMAC base; the random suffix provides uniqueness. - let client_ip = extract_client_ip(req).unwrap_or_else(|_| "unknown".to_string()); - let ec_id = generate_ec_id(settings, &client_ip)?; - log::trace!("No existing EC ID, generated: {ec_id}"); - Ok(ec_id) -} - /// Captures the EC state for a single request lifecycle. /// /// Created via [`read_from_request`](Self::read_from_request) during diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 94e99a9f..bdd7e19b 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -3,11 +3,12 @@ use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; +use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; use error_stack::{Report, ResultExt}; use fastly::http::{header, Method, StatusCode, Url}; use fastly::{Request, Response}; use serde::{Deserialize, Serialize}; -use serde_json::Value as Json; +use serde_json::{json, Value as Json}; use validator::Validate; use crate::auction::provider::AuctionProvider; @@ -416,6 +417,88 @@ fn expand_trusted_server_bidders( }) .collect() } +fn transform_prebid_response( + response: &mut Json, + request_host: &str, + request_scheme: &str, +) -> Result<(), Report> { + if let Some(seatbids) = response["seatbid"].as_array_mut() { + for seatbid in seatbids { + if let Some(bids) = seatbid["bid"].as_array_mut() { + for bid in bids { + if let Some(adm) = bid["adm"].as_str() { + bid["adm"] = json!(rewrite_ad_markup(adm, request_host, request_scheme)); + } + + if let Some(nurl) = bid["nurl"].as_str() { + bid["nurl"] = json!(make_first_party_proxy_url( + nurl, + request_host, + request_scheme, + "track" + )); + } + + if let Some(burl) = bid["burl"].as_str() { + bid["burl"] = json!(make_first_party_proxy_url( + burl, + request_host, + request_scheme, + "track" + )); + } + } + } + } + } + + Ok(()) +} + +fn rewrite_ad_markup(markup: &str, request_host: &str, request_scheme: &str) -> String { + let mut content = markup.to_string(); + let cdn_patterns = [ + ("https://cdn.adsrvr.org", "adsrvr"), + ("https://ib.adnxs.com", "adnxs"), + ("https://rtb.openx.net", "openx"), + ("https://as.casalemedia.com", "casale"), + ("https://eus.rubiconproject.com", "rubicon"), + ]; + + for (cdn_url, cdn_name) in cdn_patterns { + if content.contains(cdn_url) { + let proxy_base = format!( + "{}://{}/ad-proxy/{}", + request_scheme, request_host, cdn_name + ); + content = content.replace(cdn_url, &proxy_base); + } + } + + content = content.replace( + "//cdn.adsrvr.org", + &format!("//{}/ad-proxy/adsrvr", request_host), + ); + content = content.replace( + "//ib.adnxs.com", + &format!("//{}/ad-proxy/adnxs", request_host), + ); + content +} + +fn make_first_party_proxy_url( + third_party_url: &str, + request_host: &str, + request_scheme: &str, + proxy_type: &str, +) -> String { + let encoded = BASE64.encode(third_party_url.as_bytes()); + format!( + "{}://{}/ad-proxy/{}/{}", + request_scheme, request_host, proxy_type, encoded + ) +} + /// Copies browser headers to the outgoing Prebid Server request. /// /// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are @@ -1105,7 +1188,7 @@ impl AuctionProvider for PrebidAuctionProvider { return Ok(AuctionResponse::error("prebid", response_time_ms)); } - let response_json: Json = + let mut response_json: Json = serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { message: "Failed to parse Prebid response".to_string(), })?; @@ -1121,6 +1204,27 @@ impl AuctionProvider for PrebidAuctionProvider { } } + let request_host = response_json + .get("ext") + .and_then(|ext| ext.get("trusted_server")) + .and_then(|trusted_server| trusted_server.get("request_host")) + .and_then(|value| value.as_str()) + .unwrap_or("") + .to_string(); + let request_scheme = response_json + .get("ext") + .and_then(|ext| ext.get("trusted_server")) + .and_then(|trusted_server| trusted_server.get("request_scheme")) + .and_then(|value| value.as_str()) + .unwrap_or("https") + .to_string(); + + if request_host.is_empty() { + log::warn!("Prebid response missing request host; skipping URL rewrites"); + } else { + transform_prebid_response(&mut response_json, &request_host, &request_scheme)?; + } + let mut auction_response = self.parse_openrtb_response(&response_json, response_time_ms); self.enrich_response_metadata(&response_json, &mut auction_response); @@ -1462,6 +1566,62 @@ passphrase = "test-secret-key" ); } + #[test] + fn transform_prebid_response_rewrites_creatives_and_tracking() { + let mut response = json!({ + "seatbid": [{ + "bid": [{ + "adm": r#""#, + "nurl": "https://notify.example/win", + "burl": "https://notify.example/bill" + }] + }] + }); + + transform_prebid_response(&mut response, "pub.example", "https") + .expect("should rewrite response"); + + let rewritten_adm = response["seatbid"][0]["bid"][0]["adm"] + .as_str() + .expect("adm should be string"); + assert!( + rewritten_adm.contains("/ad-proxy/adsrvr"), + "creative markup should proxy CDN urls" + ); + + for url_field in ["nurl", "burl"] { + let value = response["seatbid"][0]["bid"][0][url_field] + .as_str() + .expect("should get tracking URL"); + assert!( + value.contains("/ad-proxy/track/"), + "tracking URLs should be proxied" + ); + } + } + + #[test] + fn make_first_party_proxy_url_base64_encodes_target() { + let url = "https://cdn.example/path?x=1"; + let rewritten = make_first_party_proxy_url(url, "pub.example", "https", "track"); + assert!( + rewritten.starts_with("https://pub.example/ad-proxy/track/"), + "proxy prefix should be applied" + ); + + let encoded = rewritten + .split("/ad-proxy/track/") + .nth(1) + .expect("should have encoded payload after proxy prefix"); + let decoded = BASE64 + .decode(encoded.as_bytes()) + .expect("should decode base64 proxy payload"); + assert_eq!( + String::from_utf8(decoded).expect("should be valid UTF-8"), + url + ); + } + #[test] fn matches_script_url_matches_common_variants() { let integration = PrebidIntegration::new(base_config()); diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 314e82ee..47e5b456 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -9,8 +9,8 @@ use fastly::{Request, Response}; use matchit::Router; use crate::constants::HEADER_X_TS_EC; -use crate::cookies::set_ec_cookie; -use crate::ec::get_or_generate_ec_id; +use crate::ec::cookies::{expire_ec_cookie, set_ec_cookie}; +use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::settings::Settings; @@ -655,33 +655,46 @@ impl IntegrationRegistry { mut req: Request, ) -> Option>> { if let Some((proxy, _)) = self.find_route(method, path) { - // Generate EC ID before consuming request - let ec_id_result = get_or_generate_ec_id(settings, &req); + // Read EC state and generate if needed before consuming request. + let ec_context = match EcContext::read_from_request(settings, &req) { + Ok(mut ec) => { + if let Err(err) = ec.generate_if_needed(settings) { + log::warn!("EC generation failed for integration proxy: {err:?}"); + } + Some(ec) + } + Err(err) => { + log::warn!("Failed to read EC context for integration proxy: {err:?}"); + None + } + }; // Set EC ID header on the request so integrations can read it. - // Header injection: Fastly's HeaderValue API rejects values containing \r, \n, or \0, - // so a crafted EC ID cannot inject additional request headers. - if let Ok(ref ec_id) = ec_id_result { - req.set_header(HEADER_X_TS_EC, ec_id.as_str()); + if let Some(ec_id) = ec_context.as_ref().and_then(|ec| ec.ec_value()) { + req.set_header(HEADER_X_TS_EC, ec_id); } let mut result = proxy.handle(settings, req).await; - // Set EC ID header on successful responses + // Consent-gated EC on successful responses: + // - Consent given → set header + cookie. + // - Consent denied + existing cookie → expire cookie (revoke). + // - Otherwise → set header only (for internal use). if let Ok(ref mut response) = result { - match ec_id_result { - Ok(ref ec_id) => { - // Response-header injection: Fastly's HeaderValue API rejects values - // containing \r, \n, or \0, so a crafted EC ID cannot inject - // additional response headers. - response.set_header(HEADER_X_TS_EC, ec_id.as_str()); - // Cookie is intentionally not set when EC ID contains RFC 6265-illegal - // characters (e.g. a crafted x-ts-ec header value). The response header - // is still emitted; only cookie persistence is skipped. - set_ec_cookie(settings, response, ec_id.as_str()); + if let Some(ref ec) = ec_context { + if let Some(ec_id) = ec.ec_value() { + response.set_header(HEADER_X_TS_EC, ec_id); + if ec.ec_allowed() { + set_ec_cookie(settings, response, ec_id); + } } - Err(ref err) => { - log::warn!("Failed to generate EC ID for integration response: {err:?}"); + if !ec.ec_allowed() { + if let Some(cookie_ec_id) = ec.existing_cookie_ec_id() { + log::info!( + "EC revoked for '{cookie_ec_id}': consent withdrawn on integration proxy" + ); + expire_ec_cookie(settings, response); + } } } } @@ -1212,7 +1225,6 @@ mod tests { } // Tests for EC ID header on proxy responses - use crate::constants::COOKIE_TS_EC; use crate::test_support::tests::create_test_settings; use fastly::http::header; @@ -1262,8 +1274,10 @@ mod tests { )]; let registry = IntegrationRegistry::from_routes(routes); - // Create a request without an EC ID cookie - let req = Request::get("https://test-publisher.com/integrations/test/ec"); + // Provide an existing EC via header (client IP is unavailable in + // the test environment, so generation would fail). + let mut req = Request::get("https://test-publisher.com/integrations/test/ec"); + req.set_header("x-ts-ec", "test-ec-id-from-header"); // Call handle_proxy (uses futures executor in test environment) let result = futures::executor::block_on(registry.handle_proxy( @@ -1274,35 +1288,29 @@ mod tests { )); // Should have matched and returned a response - assert!(result.is_some(), "Should find route and handle request"); + assert!(result.is_some(), "should find route and handle request"); let response = result.unwrap(); - assert!(response.is_ok(), "Handler should succeed"); + assert!(response.is_ok(), "handler should succeed"); let response = response.unwrap(); - // Verify x-ts-ec header is present + // The x-ts-ec header is always set for internal use by downstream + // integrations, regardless of consent. assert!( response.get_header(HEADER_X_TS_EC).is_some(), - "Response should have x-ts-ec header" + "should have x-ts-ec header on response" ); - // Verify Set-Cookie header is present (since no cookie was in request) - let set_cookie = response.get_header(header::SET_COOKIE); + // Without geo data, jurisdiction is Unknown → consent denied + // (fail-closed). The Set-Cookie header should not be set. assert!( - set_cookie.is_some(), - "Response should have Set-Cookie header for ts-ec" - ); - - let cookie_value = set_cookie.unwrap().to_str().unwrap(); - assert!( - cookie_value.contains(COOKIE_TS_EC), - "Set-Cookie should contain ts-ec cookie, got: {}", - cookie_value + response.get_header(header::SET_COOKIE).is_none(), + "should not set Set-Cookie when consent is denied" ); } #[test] - fn handle_proxy_always_sets_cookie() { + fn handle_proxy_skips_cookie_when_consent_denied() { let settings = create_test_settings(); let routes = vec![( Method::GET, @@ -1313,7 +1321,7 @@ mod tests { let registry = IntegrationRegistry::from_routes(routes); let mut req = Request::get("https://test.example.com/integrations/test/ec"); - // Pre-existing cookie + // Pre-existing cookie, but no geo data → Unknown jurisdiction → consent denied. req.set_header(header::COOKIE, "ts-ec=existing_id_12345"); let result = futures::executor::block_on(registry.handle_proxy( @@ -1326,28 +1334,22 @@ mod tests { let response = result.expect("proxy handle should succeed"); - // Should still have x-ts-ec header + // The x-ts-ec header is always set (for internal use by integrations). assert!( response.get_header(HEADER_X_TS_EC).is_some(), - "Response should still have x-ts-ec header" + "should still have x-ts-ec header for internal use" ); - // Should ALWAYS set the cookie again (per new requirements) - let set_cookie = response.get_header(header::SET_COOKIE); - + // Without geo data, jurisdiction is Unknown and consent is denied + // (fail-closed). An existing cookie should be expired (revoked). + let set_cookie = response + .get_header(header::SET_COOKIE) + .expect("should have Set-Cookie header to expire the existing cookie"); + let cookie_str = set_cookie.to_str().expect("should be valid UTF-8"); assert!( - set_cookie.is_some(), - "Should set Set-Cookie header even if cookie is present" + cookie_str.contains("Max-Age=0"), + "should expire the EC cookie when consent is denied, got: {cookie_str}" ); - - if let Some(cookie) = set_cookie { - let cookie_str = cookie.to_str().unwrap_or(""); - assert!( - cookie_str.contains(COOKIE_TS_EC), - "Should contain ts-ec cookie, got: {}", - cookie_str - ); - } } #[test] @@ -1363,8 +1365,9 @@ mod tests { )]; let registry = IntegrationRegistry::from_routes(routes); - let req = + let mut req = Request::post("https://test-publisher.com/integrations/test/ec").with_body("test body"); + req.set_header("x-ts-ec", "test-ec-id-from-header"); let result = futures::executor::block_on(registry.handle_proxy( &Method::POST, diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 66d0a6f9..7de890f8 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3,10 +3,9 @@ use fastly::http::{header, StatusCode}; use fastly::{Body, Request, Response}; use crate::backend::BackendConfig; -use crate::consent::{allows_ec_creation, build_consent_context, ConsentPipelineInput}; -use crate::constants::{COOKIE_TS_EC, HEADER_X_COMPRESS_HINT, HEADER_X_TS_EC}; -use crate::cookies::{expire_ec_cookie, handle_request_cookies, set_ec_cookie}; -use crate::ec::get_or_generate_ec_id; +use crate::constants::{HEADER_X_COMPRESS_HINT, HEADER_X_TS_EC}; +use crate::ec::cookies::{expire_ec_cookie, set_ec_cookie}; +use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::http_util::{serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; @@ -235,37 +234,21 @@ pub fn handle_publisher_request( req.get_header("x-forwarded-proto"), ); - // Parse cookies once for reuse by both consent extraction and EC ID logic. - let cookie_jar = handle_request_cookies(&req)?; - - // Capture the current EC cookie value for revocation handling. - // This must come from the cookie itself (not the x-ts-ec header) - // to ensure KV deletion targets the same identifier being revoked. - let existing_ec_cookie = cookie_jar - .as_ref() - .and_then(|jar| jar.get(COOKIE_TS_EC)) - .map(|cookie| cookie.value().to_owned()); - - // Generate EC identifiers before the request body is consumed. - // Always generated for internal use (KV lookups, logging) even when - // consent is absent — the cookie is only *set* when consent allows it. - let ec_id = get_or_generate_ec_id(settings, &req)?; - - // Extract, decode, and log consent signals (TCF, GPP, US Privacy, GPC) - // from the incoming request. The ConsentContext carries both raw strings - // (for OpenRTB forwarding) and decoded data (for enforcement). - // When a consent_store is configured, this also persists consent to KV - // and falls back to stored consent when cookies are absent. - let geo = crate::geo::GeoInfo::from_request(&req); - let consent_context = build_consent_context(&ConsentPipelineInput { - jar: cookie_jar.as_ref(), - req: &req, - config: &settings.consent, - geo: geo.as_ref(), - ec_id: Some(ec_id.as_str()), - }); - let ec_allowed = allows_ec_creation(&consent_context); - log::debug!("Proxy EC ID: {}, ec_allowed: {}", ec_id, ec_allowed,); + // Read EC state from the request (existing ID, consent, client IP). + // This must happen before the request body is consumed. + let mut ec_context = EcContext::read_from_request(settings, &req)?; + + // Generate a new EC ID if none exists and consent allows it. + // This is an organic handler, so generation is permitted here. + if let Err(err) = ec_context.generate_if_needed(settings) { + log::warn!("EC generation failed: {err:?}"); + } + + let ec_allowed = ec_context.ec_allowed(); + log::debug!( + "Proxy EC ID: {:?}, ec_allowed: {ec_allowed}", + ec_context.ec_value(), + ); let backend_name = BackendConfig::from_url( &settings.publisher.origin_url, @@ -366,17 +349,14 @@ pub fn handle_publisher_request( // - Consent absent + existing cookie → revoke (expire cookie + delete KV entry). // - Consent absent + no cookie → do nothing. if ec_allowed { - // Fastly's HeaderValue API rejects \r, \n, and \0, so the EC ID - // cannot inject additional response headers. - response.set_header(HEADER_X_TS_EC, ec_id.as_str()); - // Cookie persistence is skipped if the EC ID contains RFC 6265-illegal - // characters. The header is still emitted when consent allows it. - set_ec_cookie(settings, &mut response, ec_id.as_str()); - } else if let Some(cookie_ec_id) = existing_ec_cookie.as_deref() { + if let Some(ec_id) = ec_context.ec_value() { + response.set_header(HEADER_X_TS_EC, ec_id); + set_ec_cookie(settings, &mut response, ec_id); + } + } else if let Some(cookie_ec_id) = ec_context.existing_cookie_ec_id() { log::info!( - "EC revoked for '{}': consent withdrawn (jurisdiction={})", - cookie_ec_id, - consent_context.jurisdiction, + "EC revoked for '{cookie_ec_id}': consent withdrawn (jurisdiction={})", + ec_context.consent().jurisdiction, ); expire_ec_cookie(settings, &mut response); if let Some(store_name) = &settings.consent.consent_store { @@ -385,7 +365,7 @@ pub fn handle_publisher_request( } else { log::debug!( "EC skipped: no consent and no existing cookie (jurisdiction={})", - consent_context.jurisdiction, + ec_context.consent().jurisdiction, ); } @@ -518,22 +498,22 @@ mod tests { req.set_header("x-ts-ec", "header_id"); req.set_header("cookie", "ts-ec=cookie_id; other=value"); - let cookie_jar = handle_request_cookies(&req).expect("should parse cookies"); - let existing_ec_cookie = cookie_jar - .as_ref() - .and_then(|jar| jar.get(COOKIE_TS_EC)) - .map(|cookie| cookie.value().to_owned()); - - let resolved_ec_id = get_or_generate_ec_id(&settings, &req).expect("should resolve EC ID"); + let ec_context = + EcContext::read_from_request(&settings, &req).expect("should read EC context"); assert_eq!( - existing_ec_cookie.as_deref(), - Some("cookie_id"), - "should read revocation target from cookie value" + ec_context.ec_value(), + Some("header_id"), + "should resolve request EC ID from header precedence" + ); + assert!( + ec_context.cookie_was_present(), + "should detect cookie was present" ); assert_eq!( - resolved_ec_id, "header_id", - "should still resolve request EC ID from header precedence" + ec_context.existing_cookie_ec_id(), + Some("cookie_id"), + "should return cookie EC value for revocation, not the header value" ); } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index dabe4412..e2f65c2c 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -20,7 +20,8 @@ pub const ENVIRONMENT_VARIABLE_SEPARATOR: &str = "__"; #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] pub struct Publisher { pub domain: String, - #[validate(custom(function = validate_cookie_domain))] + /// Domain for non-EC cookies. EC cookies use a separate computed domain + /// (see [`ec_cookie_domain`](Self::ec_cookie_domain)). pub cookie_domain: String, #[validate(custom(function = validate_no_trailing_slash))] pub origin_url: String, @@ -34,6 +35,17 @@ impl Publisher { /// Known placeholder values that must not be used in production. pub const PROXY_SECRET_PLACEHOLDERS: &[&str] = &["change-me-proxy-secret"]; + /// Returns the EC cookie domain, computed as `.{domain}`. + /// + /// Per spec §5.2, EC cookies derive their domain from + /// `publisher.domain` — **not** from `publisher.cookie_domain`. + /// This ensures the EC cookie is always scoped to the publisher's + /// apex domain regardless of how `cookie_domain` is configured. + #[must_use] + pub fn ec_cookie_domain(&self) -> String { + format!(".{}", self.domain) + } + /// Returns `true` if `proxy_secret` matches a known placeholder value /// (case-insensitive). #[must_use] @@ -58,16 +70,6 @@ impl Publisher { /// }; /// assert_eq!(publisher.origin_host(), "origin.example.com:8080"); /// ``` - /// Returns the domain to use for the EC `Set-Cookie` header. - /// - /// Per spec §5.2, the EC cookie domain is `.{publisher.domain}` so - /// that the cookie is available on all subdomains of the publisher's - /// apex domain. - #[must_use] - pub fn ec_cookie_domain(&self) -> String { - format!(".{}", self.domain) - } - #[allow(dead_code)] #[must_use] pub fn origin_host(&self) -> String { @@ -534,18 +536,6 @@ impl Settings { } } -fn validate_cookie_domain(value: &str) -> Result<(), ValidationError> { - // `=` is excluded: it only has special meaning in the name=value pair, - // not within the Domain attribute value. - if value.contains([';', '\n', '\r']) { - let mut err = ValidationError::new("cookie_metacharacters"); - err.message = - Some("cookie_domain must not contain cookie metacharacters (;, \\n, \\r)".into()); - return Err(err); - } - Ok(()) -} - fn validate_no_trailing_slash(value: &str) -> Result<(), ValidationError> { if value.ends_with('/') { let mut err = ValidationError::new("trailing_slash"); @@ -737,6 +727,11 @@ mod tests { ); assert_eq!(settings.publisher.domain, "test-publisher.com"); assert_eq!(settings.publisher.cookie_domain, ".test-publisher.com"); + assert_eq!( + settings.publisher.ec_cookie_domain(), + ".test-publisher.com", + "EC cookie domain should be computed as .{{domain}}" + ); assert_eq!( settings.publisher.origin_url, "https://origin.test-publisher.com" @@ -1402,32 +1397,6 @@ mod tests { ); } - #[test] - fn test_publisher_rejects_cookie_domain_with_metacharacters() { - for bad_domain in [ - "evil.com;\nSet-Cookie: bad=1", - "evil.com\r\nX-Injected: yes", - "evil.com;path=/", - ] { - let mut settings = create_test_settings(); - settings.publisher.cookie_domain = bad_domain.to_string(); - assert!( - settings.validate().is_err(), - "should reject cookie_domain containing metacharacters: {bad_domain:?}" - ); - } - } - - #[test] - fn test_publisher_accepts_valid_cookie_domain() { - let mut settings = create_test_settings(); - settings.publisher.cookie_domain = ".example.com".to_string(); - assert!( - settings.validate().is_ok(), - "should accept a valid cookie_domain" - ); - } - /// Helper that returns a settings TOML string WITHOUT any admin handler, /// for tests that need to verify uncovered-admin-endpoint behaviour. fn settings_str_without_admin_handler() -> String { diff --git a/docs/guide/testing.md b/docs/guide/testing.md index 42eec03e..fb12ac77 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -323,7 +323,6 @@ k6 run load-test.js 5. **Use test helpers** - `create_test_settings()`, `create_test_request()` 6. **Assert specific values** - Not just `assert!(result.is_ok())` - ## Next Steps - Review [Architecture](/guide/architecture) for system design diff --git a/docs/internal/ssc-prd.md b/docs/internal/ssc-prd.md deleted file mode 100644 index 7f88ef15..00000000 --- a/docs/internal/ssc-prd.md +++ /dev/null @@ -1,976 +0,0 @@ -# Product Requirements: Edge Cookie (EC) - -**Status:** Draft -**Author:** Trusted Server Product -**Last updated:** 2026-03-12 - ---- - -## Table of Contents - -1. [Overview](#1-overview) -2. [Problem Statement](#2-problem-statement) -3. [Goals and Non-Goals](#3-goals-and-non-goals) -4. [Target Customers](#4-target-customers) -5. [TS Lite Deployment Mode](#5-ts-lite-deployment-mode) -6. [EC Identity and Cookie Structure](#6-ec-identity-and-cookie-structure) -7. [Consent Lifecycle](#7-consent-lifecycle) -8. [KV Store Identity Graph](#8-kv-store-identity-graph) -9. [Pixel Sync Endpoint](#9-pixel-sync-endpoint) -10. [S2S Batch Sync API](#10-s2s-batch-sync-api) -11. [S2S Pull Sync (TS-Initiated)](#11-s2s-pull-sync-ts-initiated) -12. [Bidstream Decoration](#12-bidstream-decoration) -13. [Configuration](#13-configuration) -14. [Documentation Updates](#14-documentation-updates) -15. [Open Questions](#15-open-questions) -16. [Success Metrics](#16-success-metrics) - ---- - -## 1. Overview - -Edge Cookie (EC) is a stable, privacy-respecting user identity mechanism built into Trusted Server. It replaces the existing SyntheticID system with a cleaner signal (IP address + publisher passphrase only), a consent-aware lifecycle, and a server-side identity graph backed by Fastly KV Store that accumulates resolved partner IDs over time. - -The EC hash is derived from the user's IP address and a publisher-chosen passphrase. A publisher's passphrase is consistent across all their own domains, producing the same EC hash for the same user everywhere they operate. Publishers may also share their passphrase with trusted partners to form an **identity-federated consortium** — members sharing a passphrase produce the same EC hash for the same user, enabling cross-property identity resolution by mutual agreement. Publishers using different passphrases produce unrelated hashes with no cross-property linkage. - -EC sets a cookie on the publisher's apex domain (e.g., `ec.publisher.com` sets `ts-ec` on `.publisher.com`) and optionally orchestrates real-time bidding or decorates outbound ad requests with resolved identity signals from configured partners. - ---- - -## 2. Problem Statement - -### 2.1 SyntheticID signal degradation - -The current SyntheticID uses User-Agent, Accept-Language, Accept-Encoding, and IP address as HMAC inputs. Each of these signals is eroding: - -- **User-Agent reduction**: Chrome's UA freeze has eliminated OS version and minor browser version. The UA string no longer meaningfully differentiates users. -- **Accept-Language homogenization**: Browser defaults increasingly converge, reducing entropy. -- **IPv6 privacy extensions**: Modern operating systems rotate the interface ID portion of IPv6 addresses on a per-session or daily basis, causing SyntheticID mismatches for returning users. - -The result is degrading match rates and false new-user rates on browsers where these signals change. - -### 2.2 No consent enforcement - -SyntheticID is created unconditionally. There is no mechanism to check TCF (EU/UK) or GPP (US) consent before creating the ID. This is a compliance gap that must be closed before EC can be offered as a product to regulated publishers. - -### 2.3 Publishers need a reliable, deterministic signal that can be explicitly shared - -Today, regular cookies don't suffice for publisher and partner needs. Additionally, only having these identifiers in the 1st party domain's cookie have created slow, undesirable behaviour in the form of cookie syncs. - ---- - -## 3. Goals and Non-Goals - -### Goals - -- Replace SyntheticID's unstable browser signal inputs with IP address + publisher salt only -- Enforce TCF and GPP consent before creating or maintaining the EC -- Implement real-time consent withdrawal: delete cookie and KV entry when consent is revoked -- Build a server-side identity graph in Fastly KV Store that accumulates resolved partner IDs over time -- Provide three KV write paths: real-time pixel sync redirects, S2S batch push from partners, and TS-initiated S2S pull from partner resolution endpoints -- Expose two bidstream integration modes: header decoration (`/identify`) and full auction orchestration (`/auction`) -- Expose a publisher-authenticated `/admin/partners/register` endpoint for partner provisioning without direct KV access - -### Non-Goals - -- Replacing the publisher's consent management platform (CMP): EC reads and enforces consent signals; it does not generate them -- Building a data management platform (DMP): EC stores resolved partner IDs as a sync spine, not audience segments -- Backward compatibility with SyntheticID: EC uses a different cookie name, header name, and ID generation method. No migration path is provided -- Real-time user matching across unrelated domains (cross-site tracking) -- Data deletion framework: out of scope for this PRD; flagged for a follow-on document -- **TS Lite deployment mode** (runtime feature flags to run EC without the full TS feature surface): requirements are captured in Section 5 but are deferred to a follow-on iteration. The current iteration targets publishers running full Trusted Server. - ---- - -## 4. Target Customers - -**This iteration** targets publishers running the full Trusted Server stack. SSP, DSP, and identity provider customers interact with EC via the sync and bidstream endpoints but do not require a separate TS deployment. - -| Customer type | Deployment mode | Primary value | In scope | -| ------------------- | --------------------------------------------------- | ------------------------------------------------------------------ | ------------------------ | -| Publisher (full TS) | Full TS + EC enabled | Consent-aware first-party ID, bidstream enrichment, identity graph | **Yes** | -| SSP | Partner — integrates via pixel sync and/or S2S pull | Build match table against EC hash; receive enriched bid requests | **Yes** (as partner) | -| DSP | Partner — integrates via S2S batch and/or S2S pull | Push/receive ID mappings; enriched bid requests | **Yes** (as partner) | -| Identity provider | Partner — integrates via S2S batch | Sync resolved IDs into the KV identity graph | **Yes** (as partner) | -| Publisher (EC only) | TS Lite at `ec.publisher.com` | First-party cookie at apex domain without full TS | Deferred (see Section 5) | - ---- - -## 5. TS Lite Deployment Mode (Deferred - out of scope) - -> **This section is out of scope for the current iteration.** Requirements are captured here for planning purposes and will be promoted to an active PRD in a follow-on phase. The current iteration delivers EC, the KV identity graph, all three sync mechanisms, and bidstream decoration — all within the existing full Trusted Server deployment model. No feature flags or route-disabling infrastructure will be built now. - -### 5.1 Concept - -TS Lite is a runtime configuration of the existing Trusted Server binary. It is not a separate binary or separate codebase. A publisher (or SSP/DSP deploying on behalf of a publisher) creates a Fastly service pointing to a subdomain — typically `ec.publisher.com` — and deploys the standard TS WASM binary with a `trusted-server.toml` that disables all routes except EC-related functionality. - -### 5.2 Route surface in TS Lite - -| Route | Full TS | TS Lite | -| -------------------------------------- | -------- | ----------------------- | -| `GET /static/tsjs=` | Enabled | Disabled | -| `POST /auction` | Enabled | Optional (configurable) | -| `GET /first-party/proxy` | Enabled | Disabled | -| `GET /first-party/click` | Enabled | Disabled | -| `POST /first-party/sign` | Enabled | Disabled | -| `GET /first-party/proxy-rebuild` | Enabled | Disabled | -| HTML injection pipeline | Enabled | Disabled | -| GTM integration | Enabled | Disabled | -| `GET /sync` | Disabled | **Enabled** | -| `GET /identify` | Disabled | **Enabled** | -| `POST /api/v1/sync` | Disabled | **Enabled** | -| `GET /.well-known/trusted-server.json` | Enabled | Enabled | - -When a disabled route is requested, TS returns `404` with the header `X-ts-error: feature-disabled`. - -### 5.3 Cookie domain and subdomain setup - -The publisher points a subdomain of their choosing (e.g., `ec`) via DNS CNAME to their Fastly service. They configure `publisher.domain = "publisher.com"` in `trusted-server.toml`. Trusted Server derives `cookie_domain = ".publisher.com"` from this setting and sets the EC cookie with that domain attribute. - -This gives the cookie read access across all subdomains of `publisher.com` — including `www.publisher.com` — without requiring a separate verification step. The publisher's control over their DNS and Fastly service implicitly proves TLD+1 ownership, following the same trust model as the existing `publisher.cookie_domain` setting. - -**Constraint:** A publisher cannot configure a cookie domain outside their declared `publisher.domain`. Attempting to set `cookie_domain = ".otherdomain.com"` is rejected at startup validation. - -### 5.4 Safari and browser compatibility - -The EC is set as an HTTP `Set-Cookie` response header (not via JavaScript). For server-set cookies on first-party publisher domains that are not classified as cross-site trackers by Safari's ITP, the effective maximum lifetime is 1 year — the same as the configured `Max-Age`. Since `ec.publisher.com` is a publisher-owned domain, it is unlikely to be classified as a tracker. - -The ITP interaction for users who arrive exclusively via third-party sync pixel redirects (where `ec.publisher.com` may be seen as a cross-site recipient) will be monitored post-launch. A cookie refresh strategy — re-issuing `Set-Cookie` on every same-site organic request — is deferred pending production data. - ---- - -## 6. EC Identity and Cookie Structure - -### 6.1 ID generation - -The EC is generated by HMAC-SHA256 of a fixed input set, using a publisher-specific secret key. - -**Inputs (IP address + salt only):** - -| Input | Value | -| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| IP address | IPv4 as-is; IPv6 summarized to /64 prefix (first 4 hextets) — discards rotating interface ID. On dual-stack, IPv6 is preferred. | -| Secret key | Publisher-chosen passphrase, configured in `trusted-server.toml`. Consistent across all of the publisher's own domains. Publishers who share the same passphrase with other publishers form an identity-federated consortium — the same user produces the same EC hash across all consortium members. Publishers using different passphrases produce unrelated hashes with no cross-property linkage. | - -**Removed from SyntheticID:** - -- `User-Agent` -- `Accept-Language` -- `Accept-Encoding` -- Handlebars template (input is now fixed, not configurable) - -**Output format (unchanged from SyntheticID):** - -``` -{64-character hex HMAC-SHA256}.{6-character random alphanumeric suffix} -``` - -The 64-character prefix is the stable, deterministic portion used as the KV store key. The 6-character suffix is random, regenerated each time a fresh EC is created. Once an EC is set in a cookie, the full value (prefix + suffix) is preserved on subsequent requests. - -**IPv6 /64 prefix rationale:** The first 64 bits of an IPv6 address identify the network prefix assigned by the ISP or home router. The remaining 64 bits (the interface ID) are rotated by privacy extensions on most modern operating systems. Using only the /64 prefix produces a stable hash for returning users while discarding the rotating portion that would cause false new-user signals. - -### 6.2 Cookie attributes - -| Attribute | Value | -| --------- | ----------------------------------------------------------------------------------------- | -| Name | `ts-ec` | -| Domain | `.publisher.com` (derived from `publisher.domain` in TOML) | -| Path | `/` | -| Secure | Yes | -| SameSite | `Lax` | -| Max-Age | `31536000` (1 year) | -| HttpOnly | No — JavaScript on `www.publisher.com` may need to read the value for ad stack decoration | - -### 6.3 Response header - -The EC value is also set as a response header for server-side consumers: - -``` -X-ts-ec: -``` - -This header is internal to Trusted Server and is stripped before proxying requests to downstream backends, consistent with how other `X-ts-*` headers are handled. - -### 6.4 Retrieval priority - -On each request, Trusted Server looks for an existing EC in this order: - -1. `X-ts-ec` request header (set by TS on a prior response, forwarded by the publisher's infrastructure) -2. `ts-ec` cookie -3. Generate fresh EC (subject to consent check — see Section 7) - -### 6.5 No backward compatibility with SyntheticID - -EC uses a different cookie name (`ts-ec` vs `synthetic_id`), a different header name (`X-ts-ec` vs `x-synthetic-id`), and a different ID generation method. No fallback to reading the `synthetic_id` cookie is provided. SyntheticID code remains in full TS and continues to function; EC is a parallel system. - ---- - -## 7. Consent Lifecycle - -Consent enforcement is a core requirement of EC. The system must not create or maintain an EC for users who have not given consent, and must actively revoke the EC when consent is withdrawn. - -### 7.1 Consent signal sources and precedence - -Section 7.1 describes **how** consent signals are read. Section 7.2 describes **whether** a signal is required at all for a given region. These two sections work in sequence: TS first determines the region (7.2), then — only if that region requires a consent signal — reads and evaluates the signal using the precedence order below. - -When a consent signal is required for the user's region, Trusted Server checks sources in the following order. The first signal found wins: - -1. **`X-consent-advertising` request header** — set by the Didomi integration (or another CMP proxy) in a prior server-side decode. This is the freshest signal and takes precedence over browser-stored values. -2. **`euconsent-v2` cookie** — the TCF v2 consent string stored by the publisher's CMP. -3. **`gpp` cookie** — the IAB Global Privacy Platform string for US state-level consent. -4. **Default: no consent** — if the region requires a signal and none is found, do not create the EC (fail safe). This step does not apply to regions where no signal is required — a user in a rest-of-world region with no consent cookies present is not subject to this fail-safe. - -### 7.2 Pre-creation consent check - -Before creating a new EC, Trusted Server first evaluates the user's region (via Fastly's `x-geo-country` header) to determine whether a consent signal is required. If the region requires a signal, TS reads it using the precedence order in Section 7.1; if no signal is found, creation is blocked (the fail-safe in step 4 applies). If the region does not require a signal, TS creates the EC unconditionally. - -| Region | Required signal | Rule | -| ---------------------------------------------------------------------------------------------------- | --------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| EU member states | TCF string | Create EC only if `purposeConsents[1]` (store and/or access information on a device) is `true`. If no TCF signal is found, do not create EC (7.1 step 4 applies). | -| United Kingdom | TCF string | Same as EU | -| US states with privacy laws (CA, CO, CT, VA, TX, OR, MT, DE, NH, NJ, TN, IN, IA, KY, NE, MD, MN, RI) | GPP string | Create EC unless user has opted out of sale or sharing of personal data. If no GPP signal is found, do not create EC (7.1 step 4 applies). | -| Rest of world | None required | Create EC on first visit regardless of whether any consent signal is present. Section 7.1 step 4 does not apply. | - -### 7.3 Consent withdrawal (real-time enforcement) - -On every request, Trusted Server decodes the consent signal (a microsecond in-memory operation with no I/O). If consent is not present or has been revoked: - -**If `ts-ec` cookie is present:** - -1. Delete the cookie by issuing `Set-Cookie: ts-ec=; Max-Age=0; Domain=.publisher.com; Path=/; Secure; SameSite=Lax` -2. Delete the KV identity graph entry: `kv_store.delete(ec_hash)` — this operation takes approximately 25ms and runs in the request path - -**If no `ts-ec` cookie is present:** - -- Do nothing - -**If consent is present:** - -- Proceed with normal EC create-or-refresh flow - -**Known tradeoff:** The KV delete adds approximately 25ms of latency to the first request after consent withdrawal. This is an intentional product decision — real-time consent enforcement is a differentiating capability of Trusted Server, and the latency cost is acceptable. - -### 7.4 Data deletion framework - -Trusted Server implements the [IAB Data Subject Rights — Data Deletion Request Framework](https://github.com/InteractiveAdvertisingBureau/Data-Subject-Rights/blob/main/Data%20Deletion%20Request%20Framework.md) as its mechanism for honoring data deletion requests from users and partners. This is the authoritative answer for partners and regulators asking "how do I delete a user?" — there is no separate interim process. - -**TS role in the framework:** Trusted Server acts as the **1st party** (it has the direct user relationship via the publisher's domain). It both receives deletion requests and initiates them downstream to registered partners who hold the same user's data. - -**How it works:** - -1. TS publishes a `dsrdelete.json` discovery file at `ec.publisher.com/.well-known/dsrdelete.json` listing its deletion endpoint, supported identifier types (EC hash), and public key. -2. A deletion request arrives as an HTTP `POST` containing a signed `rqJWT` (wrapping an `idJWT` identifying the user by EC hash). -3. TS verifies the JWT signatures, looks up the EC hash in the KV identity graph, deletes the KV entry and issues `Set-Cookie: ts-ec=; Max-Age=0` to expire the cookie. -4. TS returns a signed `acJWT` with result code `0` (success) or the appropriate error code. -5. TS propagates the deletion request to all registered partners in `partner_store` who have a resolved UID for this user, using their declared deletion endpoints. - -**Identifier type:** The EC hash (64-character hex prefix, without `.suffix`) is the stable identifier registered in `dsrdelete.json`. The `.suffix` portion is not used for deletion matching — the hash is sufficient to locate the KV entry. - -**Interim answer for partners during onboarding (before TS's deletion endpoint ships):** Publishers can manually delete a KV entry by EC hash via the Fastly KV management API or console. The EC cookie expires naturally within 1 year. A formal `POST` endpoint implementing the full JWT protocol above is required before any regulated publisher goes live. - -**Implementation status:** The `dsrdelete.json` discovery file and the JWT-based deletion endpoint are a follow-on engineering deliverable, to be completed before regulated publisher onboarding. - ---- - -## 8. KV Store Identity Graph - -### 8.1 Purpose - -The Fastly KV Store serves as a persistent identity graph keyed on the EC hash. It accumulates resolved partner IDs over time through two write paths: real-time pixel sync redirects and S2S batch pushes from partners. This graph is read at auction time to populate `user.eids` in outbound OpenRTB requests. - -### 8.2 Schema - -**KV key:** The 64-character hex hash portion of the EC (without the `.suffix`). The hash is stable across sessions for the same user+network+key combination and is safe to use as a long-lived identifier. - -**KV value (JSON body, max ~5KB):** - -```json -{ - "v": 1, - "created": 1741824000, - "last_seen": 1741910400, - "consent": { - "tcf": "CP...", - "gpp": "DBA...", - "ok": true, - "updated": 1741910400 - }, - "geo": { - "country": "US", - "region": "CA" - }, - "ids": { - "ssp_x": { "uid": "abc123", "synced": 1741824000 }, - "liveramp": { "uid": "LR_xyz", "synced": 1741890000 } - } -} -``` - -**KV metadata (max 2048 bytes, readable without streaming body):** - -```json -{ "ok": true, "country": "US", "v": 1 } -``` - -The metadata field is used for consent withdrawal checks. When consent status must be evaluated for a user with an existing EC, Trusted Server reads metadata only — not the full body — keeping the hot-path latency minimal. - -### 8.3 TTL - -KV entries are created or refreshed with a `time_to_live_sec=31536000` parameter (1 year), matching the cookie `Max-Age`. Fastly's TTL mechanism is eventual garbage collection — entries may persist up to 24 hours past expiry before being removed. This is acceptable for identity data; EC does not use KV TTL for security-critical expiration. - -### 8.4 Conflict resolution - -Concurrent writes from different partners to the same KV entry must not overwrite each other's data. Each partner's ID is stored under its own namespace in the `ids` map — a write for `ssp_x` must never clobber an existing entry for `liveramp`. Implementation must guarantee this isolation under concurrent write conditions. - -### 8.5 KV store names - -Two KV stores are required: - -| Store | TOML key | Contents | -| ---------------- | --------------- | ---------------------------------- | -| Identity graph | `ec_store` | EC hash → identity graph JSON | -| Partner registry | `partner_store` | Partner ID → config + API key hash | - -The existing `counter_store` and `opid_store` settings (currently defined but unused in `settings.rs`) can be deprecated in a follow-on cleanup. - -### 8.6 KV Store degraded behavior - -The EC cookie is deterministic (derived from IP + publisher salt) and lives in the browser. It does not depend on KV Store availability. KV Store holds identity enrichment only — resolved partner UIDs accumulated over time. The degraded behavior policy follows from this: **EC always works; enrichment degrades gracefully.** - -| Operation | KV unavailable or error | Rationale | -| --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| EC cookie creation | Set the cookie. Skip the KV entry creation silently. Log the failure at `warn` level. | The cookie is the identity anchor — it does not require KV. The KV entry will be created on the next request once KV recovers. | -| EC cookie refresh (existing user) | Refresh the cookie. Skip the KV `last_seen` update silently. Log at `warn`. | Same as above — the cookie continues working. Stale `last_seen` is acceptable. | -| `/sync` KV write | Redirect to `return` with `ts_synced=0&ts_reason=write_failed`. | The browser redirect must not be blocked by KV availability. This case is already specified in Section 9.4. | -| `/identify` KV read | Return `200` with `ec` hash (from cookie) and `degraded: true`. Set `uids: {}` and `eids: []`. | The EC hash is still valid and useful for attribution and analytics. Empty uids signal that enrichment is unavailable, not that the user has no synced partners. `degraded: true` lets callers distinguish transient KV failure from a genuinely unenriched user. | -| S2S batch write (`/api/v1/sync`) | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | The request was valid; the failure is infrastructure. Partners should retry the batch. | -| S2S pull sync write (async) | Discard the resolved uid. Log at `warn`. Retry will occur on the next qualifying request per the `pull_sync_ttl_sec` window. | Async path — no user-facing impact. | -| Consent withdrawal KV delete | Expire the cookie immediately. Log the KV delete failure at `error` level. Retry the KV delete on the next request for this user. | Cookie deletion is the primary enforcement mechanism. KV delete failure must not block or delay the cookie expiry. | - -**`degraded: true` in `/identify` responses** - -When a KV read fails, the `/identify` response includes `"degraded": true` in the JSON body alongside an empty `uids` and `eids`. The `ec` field is still populated from the cookie. Callers should proceed with identity-only targeting (EC hash) and omit partner UID parameters from downstream requests. - -```json -{ - "ec": "a1b2c3...AbC123", - "consent": "ok", - "degraded": true, - "uids": {}, - "eids": [] -} -``` - -### 8.7 Buyer confidence in KV entries (Deferred - out of scope) - -#### Problem - -Code attestation (reproducible WASM builds + published binary hashes) proves that the TS binary running on Fastly's infrastructure matches the open-source repository. It does not, however, prove that the _data_ inside `ec_store` was written by that attested binary. A malicious or compromised operator could write arbitrary identity mappings directly into the KV store — bypassing all code paths — and buyers would have no way to detect it. - -#### Solution: JOSE-signed KV entry bodies - -Every identity graph entry written to `ec_store` by the TS WASM binary is signed using JSON Web Signatures (JWS, RFC 7515) before storage. The signing key is generated at binary load time and is bound to the running instance; the corresponding public key is published alongside the binary hash in the attestation record. - -At read time, the TS binary verifies the JWS signature before consuming any fields from the entry. An entry that fails signature verification is treated as absent, the request proceeds as if the KV key does not exist, and the failure is logged at `error` level. - -**What a valid signature proves:** - -- The entry was written by a TS binary instance whose signing key corresponds to a published, attested binary hash. -- The entry body has not been modified since it was written. -- A buyer who trusts the attested binary can transitively trust any entry that carries a valid signature. - -**What it does not prove:** - -- That the _input data_ (e.g., a partner-supplied UID) was accurate at the time of write. Signal accuracy remains the partner's responsibility. -- Anything about entries written before this feature was deployed. A migration pass will resign existing entries or treat them as unsigned (degraded) until they are refreshed by a normal TS write. - -#### Attestation record endpoint - -The signing public key is published as a namespaced field inside the existing `/.well-known/trusted-server.json` discovery document — the same endpoint partners already fetch for request signing key distribution. No new endpoint is required. - -``` -GET /.well-known/trusted-server.json -``` - -Response (application/json): - -```json -{ - "version": "1.0", - "jwks": { - "keys": [ - { - "kty": "OKP", - "crv": "Ed25519", - "kid": "ts-2026-A", - "use": "sig", - "x": "..." - } - ] - }, - "attestation": { - "binary_hash": "", - "alg": "ES256", - "jwk": { "kty": "EC", "crv": "P-256", "x": "...", "y": "..." }, - "expires_at": "2026-06-18T00:00:00Z" - } -} -``` - -The `jwks` field is unchanged — it continues to serve request signing keys on its existing rotation schedule. The `attestation` object is a separate namespace and does not affect existing consumers of this endpoint. - -| Field | Description | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | -| `attestation.binary_hash` | SHA-256 hex of the deployed WASM binary. Cross-referenced with Fastly's signed deployment manifest in the reproducible builds PRD. | -| `attestation.alg` | JWS algorithm used for all KV entry signatures. Fixed at `ES256` (ECDSA P-256). | -| `attestation.jwk` | Public key in JWK format (RFC 7517). Buyers use this to verify signatures in KV-derived `user.eids`. Distinct from the `jwks` request-signing keys. | -| `attestation.expires_at` | UTC timestamp after which the attestation key should be considered untrustworthy. Buyers must re-fetch before this time. | - -**Key TTL:** 90 days. The attestation key rotates on each new TS deployment. The previous key's `expires_at` is set 7 days after rotation to allow in-flight impressions to drain. - -**Key storage:** The signing private key lives in the Fastly Secret Store under `ec_signing_key`. It is provisioned at deploy time and never exposed in responses or logs. - -**Caching:** `trusted-server.json` should be served with `Cache-Control: max-age=3600` to ensure buyers pick up a rotated attestation key within one hour of a new deployment. This is shorter than the JWKS key rotation window and is safe for both key types. - -> **Future:** When the reproducible builds PRD ships, the `attestation` object may be graduated to a dedicated `/.well-known/ts-attestation.json` endpoint if the data (multiple binary hashes, Fastly co-signatures) outgrows the shared document. The field names will remain compatible. - -#### Buyer-facing verification flow - -1. Publisher includes `site.ext.ts_discovery` pointing to `/.well-known/trusted-server.json` in the bid request. -2. Buyer fetches `trusted-server.json` and caches it until `attestation.expires_at`. -3. Buyer independently verifies `attestation.binary_hash` against Fastly's signed deployment manifest (see separate PRD). -4. Buyer verifies the JWS signature on each `user.eids` entry against `attestation.jwk`. -5. Buyer trusts KV-derived signals only for entries with a valid signature from a non-expired attestation key. - -#### Relationship to reproducible builds PRD - -JOSE-signed KV entries close the _data integrity_ gap that code attestation leaves open. Reproducible builds and published binary hashes address the _code integrity_ layer — proving that the deployed binary matches the audited source. These are complementary controls that together form a complete trust chain for buyers. - -The reproducible builds feature has broader scope than the identity graph (it applies to all TS behaviour, not just KV writes) and will be specified in a dedicated PRD. The `attestation.binary_hash` field in `trusted-server.json` anticipates that PRD — buyers can record it today, and the reproducible builds PRD will define the process for independently verifying it against Fastly's signed deployment manifest. - ---- - -## 9. Pixel Sync Endpoint - -### 9.1 Purpose - -The pixel sync endpoint allows SSPs and DSPs to synchronize their user IDs with the EC hash via a browser-side redirect. When a partner's sync pixel fires, the user's browser is redirected through `ec.publisher.com/sync`, Trusted Server reads the existing `ts-ec` cookie, and writes the partner's user ID into the KV identity graph. - -This is the primary real-time write path for building the identity graph from existing cookie sync infrastructure. - -### 9.2 Endpoint - -``` -GET /sync -``` - -### 9.3 Parameters - -| Parameter | Required | Description | -| --------- | -------- | ------------------------------------------------------------------------------------------------- | -| `partner` | Yes | Partner ID, must match a registered partner in `partner_store` KV | -| `uid` | Yes | Partner's user ID for this user | -| `return` | Yes | Callback URL to redirect to after sync (must match partner's `allowed_return_domains`) | -| `consent` | No | TCF or GPP string from the partner's context, used if no consent signal is present on the request | - -### 9.4 Flow - -1. Read the `ts-ec` cookie. If absent, redirect to `return` URL with `ts_synced=0` appended. Do not create a new EC during a sync — a sync redirect is not an organic user visit and must not be used to bootstrap identity. -2. Look up the partner record in `partner_store` KV using the `partner` parameter. Return `400` if the partner is not found. -3. Validate the `return` URL against the partner's `allowed_return_domains`. Return `400` if the domain is not on the allowlist. -4. Evaluate consent for this user by decoding from request cookies (or the optional `consent` query parameter if no cookie signal is present). If consent is absent or invalid, redirect to `return` with `ts_synced=0&ts_reason=no_consent`. No KV write is performed. -5. Perform an atomic read-modify-write to update `ids[partner_id]` in the KV identity graph (with generation marker — see Section 8.4). If the write fails after all retries, redirect to `return` with `ts_synced=0&ts_reason=write_failed`. -6. On successful KV write, redirect to `return` with `ts_synced=1` appended as a query parameter. - -**`ts_synced` values:** - -| Value | Meaning | -| ------------------------------------ | --------------------------------------------------------------------------- | -| `ts_synced=1` | KV write succeeded — partner uid is now in the identity graph | -| `ts_synced=0&ts_reason=no_ec` | No EC cookie present — user has not established an EC on this publisher | -| `ts_synced=0&ts_reason=no_consent` | Consent absent or invalid — write suppressed | -| `ts_synced=0&ts_reason=write_failed` | KV write failed after retries — partner should retry on a future pixel fire | - -Partners should treat `ts_synced=0` as a signal that the mapping was not stored. The `ts_reason` parameter is informational; partners should not gate their own behavior on specific reason values. - -### 9.5 Security - -- The `return` URL is validated against the partner's `allowed_return_domains` using **exact hostname match** — `sync.example-ssp.com` does not match `a.sync.example-ssp.com`. Suffix or wildcard matching is not supported. This prevents subdomain takeover abuse where an attacker controlling an abandoned subdomain of a legitimate partner could exploit TS as an open redirect. Partners needing multiple callback hostnames must register each one explicitly in `allowed_return_domains`. Open redirects are not permitted. -- Partners control when to fire their sync pixel; no HMAC signature is required on the inbound sync request. -- Anti-stuffing rate limit: a maximum of `sync_rate_limit` sync writes per EC hash per hour per partner (configurable per partner in `partner_store`, default 100). - -### 9.6 User stories - -**As an SSP**, I want to fire a sync pixel when I see a user so that I can associate my user ID with the EC hash and receive enriched bid requests when the publisher calls Trusted Server for auction. - -**Acceptance criteria:** - -- [ ] `GET /sync?partner=ssp_x&uid=abc&return=https://sync.ssp.com/ack` returns a redirect to the `return` URL within 50ms (excluding KV write time) -- [ ] KV entry for the EC hash contains `ids.ssp_x.uid = "abc"` after a successful sync; response redirects to `return` with `ts_synced=1` -- [ ] If no `ts-ec` cookie is present, redirects to `return` with `ts_synced=0&ts_reason=no_ec`; no KV write performed -- [ ] If consent is absent or invalid, redirects to `return` with `ts_synced=0&ts_reason=no_consent`; no KV write performed -- [ ] If KV write fails after all retries, redirects to `return` with `ts_synced=0&ts_reason=write_failed` -- [ ] `return` URL domains not in partner's `allowed_return_domains` receive a `400` response (no redirect) -- [ ] Rate limit is enforced: more than `sync_rate_limit` writes per hour per EC hash per partner are rejected with `429` - ---- - -## 10. S2S Batch Sync API - -### 10.1 Purpose - -The S2S batch sync API allows partners to push ID mappings to Trusted Server in bulk via an authenticated REST endpoint. This write path handles large-scale partner-initiated syncs, back-fills for users whose browser-side pixel sync has not fired, and DSP-side match data that originates from non-browser contexts. - -### 10.2 Endpoint - -``` -POST /api/v1/sync -``` - -### 10.3 Authentication - -Partners authenticate with a rotatable API key. Key rotation must not require redeploying the binary. Partner provisioning is handled via the `/admin/partners/register` endpoint (see Section 15, Open Questions). - -### 10.4 Request - -``` -POST /api/v1/sync -Content-Type: application/json -Authorization: Bearer - -{ - "mappings": [ - { - "ec_hash": "<64-character hex hash>", - "partner_uid": "abc123", - "timestamp": 1741824000 - }, - ... - ] -} -``` - -Maximum batch size per request: 1000 mappings (subject to revision based on KV write throughput testing). - -### 10.5 Response - -```json -{ - "accepted": 998, - "rejected": 2, - "errors": [ - { "index": 45, "reason": "ec_hash_not_found" }, - { "index": 72, "reason": "consent_withdrawn" } - ] -} -``` - -**HTTP status rules:** - -| Condition | Status | -| ------------------------------------------------------ | --------------------------------------- | -| All mappings accepted | `200 OK` | -| Some mappings accepted, some rejected | `207 Multi-Status` | -| Auth valid, batch valid, but **all** mappings rejected | `207 Multi-Status` with `accepted: 0` | -| Auth invalid | `401 Unauthorized` (no body processing) | -| Batch exceeds 1000 mappings or malformed JSON | `400 Bad Request` (no body processing) | - -A `207` with `accepted: 0` signals "your request was received and processed correctly, but none of the submitted EC hashes were found or eligible." This is distinct from an auth or protocol error. Partners should treat this as a data signal — either the EC hashes are stale/unknown, or consent has been withdrawn for all submitted users — and should not retry the same batch without investigating the underlying cause. - -### 10.6 Consent enforcement - -Before writing a mapping, Trusted Server checks the KV metadata for the given EC hash. Mappings for users with `consent.ok = false` are rejected with reason `consent_withdrawn`. Partners must not submit mappings for users who have withdrawn consent; this enforcement is a safeguard, not the primary compliance mechanism. - -### 10.7 Conflict resolution - -- If the KV entry does not exist for a given `ec_hash`, the mapping is rejected with reason `ec_hash_not_found`. The S2S API does not create new KV entries — only the EC creation flow (from organic browser visits) can create entries. -- If the partner has an existing entry for the same `ec_hash` and the request's `timestamp` is older than the stored `synced` timestamp, the mapping is skipped (no error, counted as accepted). -- Otherwise, atomic read-modify-write with generation markers (see Section 8.4). - -### 10.8 User stories - -**As a DSP**, I want to push my user ID mappings to Trusted Server in bulk so that the publisher's auction requests are enriched with my resolved ID and I can bid on users I recognize. - -**Acceptance criteria:** - -- [ ] `POST /api/v1/sync` with a valid Bearer token and a batch of up to 1000 mappings returns a response within 5 seconds -- [ ] Accepted mappings are written to the corresponding KV identity graph entries within 1 second -- [ ] Mappings for unknown `ec_hash` values are rejected with `ec_hash_not_found` -- [ ] Mappings for users with withdrawn consent are rejected with `consent_withdrawn` -- [ ] Invalid or expired Bearer tokens receive `401 Unauthorized` -- [ ] Requests exceeding 1000 mappings receive `400 Bad Request` -- [ ] Rate limiting by API key is enforced - ---- - -## 11. S2S Pull Sync (TS-Initiated) - -### 11.1 Purpose - -The pixel sync endpoint (Section 9) requires the user's browser to initiate a redirect, which can be blocked by ad blockers or ITP. The S2S batch API (Section 10) requires the partner to proactively push mappings. Neither path helps when the publisher wants to opportunistically ask a partner "do you know this user?" without waiting for a pixel to fire. - -S2S pull sync inverts the S2S batch model: Trusted Server calls the partner's resolution endpoint directly, server-to-server, and writes the returned uid into the KV identity graph. No browser pixel is involved. The HTTP return path is the response body — no redirect required. - -**What the partner resolves against** - -The partner's resolution endpoint receives the EC hash and IP address. The partner must look these up against their own **server-side user database** — not a browser cookie. Common sources partners use: - -- **IP-based user graph**: major SSPs and DSPs maintain server-side mappings of IP → their own uid, built from bid stream traffic and direct visits. If a user has hit any page on which this partner runs, they may have an IP mapping. -- **Prior bid stream observation**: once the EC hash begins appearing in outbound bid requests (Mode B), partners who have bid on those requests can build their own reverse map of EC hash → their uid. Subsequent pull calls can then be resolved against this map. -- **Authenticated / hashed-email graph**: for partners with deterministic identity (UID2, RampID), they may resolve from email-hash mappings independently of IP. - -**Implication:** pull sync only returns a uid for users the partner already knows by some server-side signal. If the partner has never seen this user by any channel, they return null and the call is a no-op. This is not a general solution for new users — it is a reliable, pixel-free path for users the partner already knows. - -**What it solves and what it doesn't:** - -| User scenario | Pixel sync | S2S batch | S2S pull | -| ----------------------------------------- | ------------------------- | ------------------------------ | -------------------------------------------- | -| New user, Chrome, 3p cookies available | Works (bootstraps KV) | Not applicable | No server-side mapping yet — no-op | -| Returning user after prior pixel sync | Redundant (already in KV) | Works | Works (partner has IP or bid-stream mapping) | -| Safari user, partner has IP-based mapping | Blocked / unreliable | Works if partner knows EC hash | Works — partner resolves from their IP graph | -| User unknown to partner by any signal | No uid to sync | No uid to push | No uid to return — no-op | -| Authenticated user with hashed email | Works | Works | Works | - -S2S pull does not solve the cold-start problem for users the partner has never seen. It degrades gracefully to a no-op in those cases. - -### 11.2 When TS initiates a pull - -Trusted Server initiates a pull sync for a given partner when all of the following are true on an incoming request: - -1. A valid `ts-ec` cookie is present (user has an established EC) -2. Consent is valid for this user -3. The partner has `pull_sync_enabled: true` in their `partner_store` record -4. The KV identity graph for this EC hash has no entry for this partner, **or** the existing entry's `synced` timestamp is older than `pull_sync_ttl_sec` (configurable per partner, default 86400 — 1 day) - -### 11.3 Execution model - -Pull sync calls are dispatched **asynchronously after the response is sent** using Fastly's `send_async` / background task model. They do not add latency to the user-facing request. - -A maximum of `pull_sync_concurrency` partner calls are dispatched per request (configurable globally, default 3). If more partners qualify, they are queued and dispatched on subsequent requests for the same user. - -### 11.4 Partner resolution endpoint - -Each partner exposes a resolution endpoint declared in their `partner_store` record as `pull_sync_url`. Trusted Server calls it with a `GET` request: - -``` -GET ?ec_hash=<64-char-hex>&ip= -Authorization: Bearer -``` - -`ts_pull_token` is a per-partner token provisioned during partner registration, used so the partner can authenticate inbound requests from Trusted Server. It is stored in `partner_store` KV in plaintext (outbound credential, not inbound). - -**Expected response (`200 OK`):** - -```json -{ "uid": "abc123" } -``` - -**If the partner does not recognize the user:** - -```json -{ "uid": null } -``` - -or `404 Not Found`. Both are treated as a no-op — no KV write. - -Any response other than `200` with a valid body is treated as a transient failure. Trusted Server does not retry on failure; the next qualifying request for this user will trigger a new attempt. - -### 11.5 KV write - -On a successful resolution (`uid` is non-null), Trusted Server performs the same atomic read-modify-write used by the pixel sync path (Section 8.4): read the existing KV entry with a generation marker, merge `ids[partner_id].uid`, write back with `if-generation-match`. - -The `synced` timestamp is set to the current Unix timestamp, which resets the `pull_sync_ttl_sec` clock. - -### 11.6 Partner configuration additions - -The following fields are added to the partner record schema (Section 13.3): - -```json -{ - "pull_sync_enabled": true, - "pull_sync_url": "https://api.example-ssp.com/ts/resolve", - "pull_sync_ttl_sec": 86400, - "ts_pull_token": "" -} -``` - -### 11.7 Security - -- The `pull_sync_url` domain must be on an allowlist declared in the partner record. Trusted Server will not call arbitrary URLs. -- Pull sync calls are one-way data flows: TS sends only the EC hash and IP. No other user data (consent string, geo, other partner IDs) is included in the pull request. -- Rate limiting: a maximum of `pull_sync_rate_limit` pull calls per EC hash per partner per hour (configurable per partner, default 10). This prevents the pull mechanism from being used as a polling channel. - -### 11.8 User stories - -**As an SSP**, I want Trusted Server to call my resolution endpoint when it sees a user I might know, so that my uid is available for bidstream decoration without requiring the publisher to include a sync pixel in their page. - -**Acceptance criteria:** - -- [ ] When a request arrives with a valid `ts-ec` cookie and a partner with `pull_sync_enabled: true` has no KV entry (or a stale entry), a pull call is dispatched asynchronously after the response is sent -- [ ] A successful pull response with a non-null `uid` results in a KV write within 1 second -- [ ] A `null` or `404` response results in no KV write and no error logged above `DEBUG` level -- [ ] Pull calls are not initiated during the pixel sync flow (no double-write) -- [ ] Rate limit is enforced: more than `pull_sync_rate_limit` pull calls per EC hash per partner per hour are suppressed -- [ ] Pull calls do not add measurable latency to the user-facing response (async dispatch) - ---- - -## 12. Bidstream Decoration - -### 12.1 Two integration modes - -Trusted Server exposes two modes for injecting EC identity into the bidstream. Publishers choose the mode that fits their existing ad stack. - -### 12.2 Mode A: Identity resolution (`/identify`) - -Trusted Server exposes `/identify` as a standalone identity resolution endpoint for callers that need EC identity and resolved partner UIDs outside of TS's own auction orchestration. TS builds the OpenRTB request in Mode B — `/identify` is not part of that path. It serves three distinct use cases: - -**Use case 1 — Attribution and analytics** -Any server-side or browser-side system that needs to tag an event, impression, or conversion with the user's EC hash. Examples: analytics pipelines, attribution platforms, reporting dashboards. - -**Use case 2 — Publisher ad server outbid context** -After TS's auction completes and winners are delivered to the publisher's ad server endpoint, the publisher's ad server may need EC identity and resolved partner UIDs to evaluate whether to accept the programmatic winner or outbid with a direct-sold placement. For this use case, TS includes the EC identity in the winner notification payload directly (see Section 12.3) — a separate `/identify` call is only needed if the publisher's ad server receives the winner through a path that does not carry TS headers. - -**Use case 3 — Client-side wrappers for non-TS SSPs** -Some SSPs run client-side header bidding wrappers (e.g., Amazon TAM, certain Index Exchange configurations) that do not participate in TS's server-side auction orchestration. A Prebid.js module or custom wrapper script calls `/identify` from the browser to obtain the EC hash and resolved partner UIDs, then injects those values into bid requests sent to those SSPs. This ensures non-TS demand sources bid with the same identity enrichment as TS-orchestrated bids, enabling a fair comparison at winner selection. - -> **Prerequisite for use case 3:** For a non-TS SSP to receive a useful UID from `/identify`, that SSP must already be a registered partner in `partner_store` and must have a resolved uid in the KV identity graph for this user (via pixel sync, S2S batch, or S2S pull). Without a prior sync, `/identify` returns no uid for that partner. - -**Endpoint:** `GET /identify` - -**When to call:** Once per auction event — not per-pageview. For use case 3, call before sending bid requests to non-TS SSPs. - -#### Call patterns - -**Pattern 1 — Browser-direct (recommended for use cases 1 and 3)** - -A script on the publisher's page calls `/identify` via `fetch()`. Because `ec.publisher.com` is same-site with the publisher's domain, the browser sends the `ts-ec` cookie and consent cookies automatically. No forwarding required. - -```js -const identity = await fetch('https://ec.publisher.com/identify').then((r) => - r.json() -) - -// GAM key-value targeting -googletag.pubads().setTargeting('ts_ec', identity.ec) -googletag.pubads().setTargeting('ts_uid2', identity.uids.uid2) - -// Prebid.js userIds injection -pbjs.setConfig({ - userSync: { userIds: [{ name: 'uid2', value: { id: identity.uids.uid2 } }] }, -}) -``` - -**Pattern 2 — Origin-server proxy (for use case 2 when TS winner headers are unavailable)** - -A server-side caller must forward the following from the original browser request: - -| Header to forward | Required | -| ------------------------------------------------------- | --------------------------------------------------------------------- | -| `Cookie: ts-ec=` or `X-ts-ec: ` | Yes — without this, TS cannot identify the user | -| `Cookie: euconsent-v2=` or `Cookie: gpp=` | Yes — without this, TS returns `consent: denied` and no identity data | -| `X-consent-advertising: ` | Optional — takes precedence over cookie consent if present | - -#### Cookie and consent handling - -`/identify` follows the EC retrieval priority from Section 6.4. It does not generate a new EC — if no EC is present, the response body contains `consent: denied` and empty identity fields. Consent is evaluated per Section 7.1. `/identify` never sets or modifies cookies. - -#### Response - -**`200 OK` — identity resolved** - -EC is present and consent is valid. Identity values are returned as a JSON body. Callers use these values to construct URL parameters for GAM, SSP bid requests, analytics events, or any other downstream system. - -```json -{ - "ec": "a1b2c3...AbC123", - "consent": "ok", - "degraded": false, - "uids": { - "uid2": "A4A...", - "liveramp": "LR_xyz", - "id5": "ID5-abc" - }, - "eids": [ - { "source": "uidapi.com", "uids": [{ "id": "A4A...", "atype": 3 }] }, - { "source": "liveramp.com", "uids": [{ "id": "LR_xyz", "atype": 3 }] } - ] -} -``` - -`uids` contains one key per partner with `bidstream_enabled: true` and a resolved UID in the KV graph. Partners with no resolved UID for this user are omitted. - -**`200 OK` — KV unavailable (degraded)** - -EC is present and consent is valid, but the KV read failed. The EC hash is returned; `uids` and `eids` are empty. `degraded: true` distinguishes this from a user who simply has no synced partners yet — callers should proceed with EC-only targeting and may retry on the next auction. - -```json -{ - "ec": "a1b2c3...AbC123", - "consent": "ok", - "degraded": true, - "uids": {}, - "eids": [] -} -``` - -**`403 Forbidden` — consent denied** - -EC is present but the user has not given consent (or consent has been withdrawn). Callers must omit identity parameters from all downstream requests. The status code alone is sufficient to detect this case — body parsing is not required. - -```json -{ "consent": "denied" } -``` - -**`204 No Content` — no EC present** - -No `ts-ec` cookie and no `X-ts-ec` header was found on the request. The user has not yet established an EC on this publisher. No body is returned. Callers should proceed without identity enrichment. - -#### Response headers (supplementary) - -In addition to the JSON body, TS sets the following response headers for server-to-server callers, logging, and future use. These are not the primary integration contract — callers should read the JSON body. - -| Header | Value | -| ------------------- | ------------------------------------------------------------- | -| `X-ts-ec` | `` or absent if no EC | -| `X-ts-eids` | Base64-encoded JSON array of OpenRTB 2.6 `user.eids` objects | -| `X-ts-` | Resolved UID per partner (e.g., `X-ts-uid2`, `X-ts-liveramp`) | -| `X-ts-ec-consent` | `ok` or `denied` | - -### 12.3 Mode B: Full auction orchestration (`/auction`) - -Trusted Server owns the full auction path in Mode B. TS builds the OpenRTB request, injects EC identity and resolved partner UIDs, sends it to Prebid Server, receives bids, selects winners, and delivers the winner set to the publisher's ad server endpoint. The publisher's ad server does not build the OpenRTB request — it receives auction winners from TS and either accepts the programmatic winner or outbids it with a direct-sold placement. - -**EC injection into the outbound OpenRTB request (changes from current behavior):** - -- `user.id` is set to the full EC value (`hash.suffix`) -- `user.eids` is populated from the KV identity graph for this user (see OpenRTB structure below) -- `user.consent` is set to the decoded TCF string (currently always `null`) -- SSP-specific `ext.eids`: when calling a specific PBS adapter, only that SSP's resolved ID is included in the adapter-level `ext.eids`. All configured identity providers are included at the top-level `user.eids`. - -**EC context in winner notification to publisher's ad server:** - -When TS delivers auction winners to the publisher's ad server endpoint, the response includes EC identity so the publisher's ad server has full context for its outbid decision without needing to call `/identify` separately: - -| Header | Value | -| ----------------- | ------------------------------------------------------------ | -| `X-ts-ec` | `` | -| `X-ts-eids` | Base64-encoded JSON array of OpenRTB 2.6 `user.eids` objects | -| `X-ts-ec-consent` | `ok` or `denied` | - -### 12.4 OpenRTB 2.6 `user.eids` structure - -```json -{ - "user": { - "id": "a1b2c3...AbC123", - "consent": "CP...", - "eids": [ - { - "source": "liveramp.com", - "uids": [{ "id": "LR_xyz", "atype": 3 }] - }, - { - "source": "id5-sync.com", - "uids": [{ "id": "ID5-abc", "atype": 3 }] - }, - { - "source": "uidapi.com", - "uids": [{ "id": "A4A...", "atype": 3 }] - } - ] - } -} -``` - -`atype` values follow the OpenRTB 2.6 specification: `1` = cookie/device, `2` = hashed email, `3` = partner-defined. All EC-derived IDs use `atype: 3`. - -### 12.5 Partner taxonomy - -Each partner registered in `partner_store` declares: - -- `source_domain`: the OpenRTB `source` value for their EID (e.g., `"liveramp.com"`) -- `openrtb_atype`: integer (typically `3`) -- `bidstream_enabled`: boolean — whether this partner's UID should appear in `user.eids` on auction requests - -### 12.6 User stories - -**As a publisher using Mode A for analytics/attribution**, I want to call `/identify` from a browser script so that I can tag events and impressions with the user's EC hash and resolved partner UIDs using URL parameters. - -**Acceptance criteria:** - -- [ ] `GET /identify` returns `200` with a valid JSON body within 30ms when EC is present and consent is valid -- [ ] `uids` object contains one key per partner with `bidstream_enabled: true` and a resolved UID; partners with no resolved UID are omitted -- [ ] If consent is denied, response is `403 Forbidden` with body `{"consent": "denied"}` -- [ ] If no EC is present, response is `204 No Content` with no body -- [ ] Response headers `X-ts-ec`, `X-ts-eids`, `X-ts-`, and `X-ts-ec-consent` are present on `200` responses as supplementary signals - -**As a publisher using a client-side wrapper for non-TS SSPs**, I want to call `/identify` from my Prebid.js configuration so that SSPs outside TS's auction receive the same identity enrichment as TS-orchestrated bids, enabling a fair winner comparison. - -**Acceptance criteria:** - -- [ ] `GET /identify` called from the browser returns resolved UIDs for all registered partners with a KV entry for this user -- [ ] A partner with no KV entry for this user is omitted from `uids` — no empty or null entries -- [ ] Response is available within 30ms so it does not block Prebid.js auction timeout - -**As a publisher using Mode B**, I want Trusted Server to build and send enriched OpenRTB requests to Prebid Server and deliver winners to my ad server with full EC context, so my ad server can make outbid decisions without additional identity lookups. - -**Acceptance criteria:** - -- [ ] Outbound OpenRTB request to PBS contains `user.id` equal to the EC value -- [ ] `user.eids` contains one entry per partner with `bidstream_enabled: true` and a resolved UID in the KV graph -- [ ] `user.consent` contains the decoded TCF string when available -- [ ] Partners without a resolved UID for this user are omitted from `user.eids` (no empty entries) -- [ ] Winner notification to publisher's ad server includes `X-ts-ec`, `X-ts-eids`, and `X-ts-ec-consent` headers - ---- - -## 13. Configuration - -The following capabilities must be configurable without redeploying the binary: - -- **EC enable/disable** — EC can be turned on or off per deployment -- **Publisher passphrase** — the HMAC key used for EC hash generation; same value across all of the publisher's domains; shared with trusted partners to form an identity-federated consortium -- **Identity graph store** — the KV store backing the EC hash → identity graph -- **Partner registry store** — the KV store backing partner configuration and API key validation -- **Partner records** — each partner's allowed sync domains, bidstream settings, pull sync configuration, and API credentials; managed via `/admin/partners/register` without redeployment - -The exact configuration format (TOML keys, KV schema, JSON field names) is an engineering decision and will be documented in the technical design doc. - ---- - -## 14. Documentation Updates - -The following documentation changes are required alongside the EC feature: - -- **Rename SyntheticID → Edge Cookie** across the entire `docs/` GitHub Pages site. The underlying concept is the same but the product name changes. -- **New integration guides**, one per customer type: - - Publisher (full TS): enabling EC in `trusted-server.toml`, partner onboarding via `/admin/partners/register` - - SSP: pixel sync integration guide, sync pixel URL format, callback handling, optional pull resolution endpoint - - DSP: S2S batch API reference, authentication, conflict resolution behavior, optional pull resolution endpoint - - Identity Provider: registering as a partner, `source_domain` and `openrtb_atype` configuration, sync patterns -- **API reference** for the four new endpoints: `GET /sync`, `GET /identify`, `POST /api/v1/sync`, and the partner-side pull resolution contract -- **Pull sync integration guide**: partner requirements for exposing a resolution endpoint, authentication, expected response shape, rate limit behavior -- **Consent enforcement guide**: how TCF and GPP signals are read, precedence rules, what happens on withdrawal - ---- - -## 15. Open Questions - -| # | Question | Owner | Status | -| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | --------------------------------------------------------------------------- | -| 1 | Partner provisioning: TS will expose a `/admin/partners/register` endpoint authenticated at the publisher level (bearer token issued per publisher Fastly service), so publishers can onboard SSP/DSP partners without touching KV directly. Engineering to define the exact auth mechanism. | Engineering | **Resolved** — `/admin/partners/register` endpoint, publisher-authenticated | -| 2 | Should TS Lite expose a `GET /health` endpoint so partners can programmatically verify their service is running and their partner config is active in KV? | Product | **N/A** — TS Lite deferred (see Section 5) | - ---- - -## 16. Success Metrics - -| Metric | Target | Measurement method | -| ------------------------------- | --------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | -| EC match rate (returning users) | >90% within 30 days | Fastly real-time logs: ratio of requests with existing `ts-ec` cookie vs. new EC generations | -| Consent enforcement accuracy | 0 ECs created for opted-out EU/UK users | Log audit: verify no `ts-ec` `Set-Cookie` in responses where consent signal is absent | -| KV sync latency (pixel sync) | p99 <75ms end-to-end | Fastly log timing on `/sync` endpoint | -| S2S batch API throughput | >500 mappings/sec sustained | Load test prior to partner onboarding | -| S2S pull sync resolution rate | >30% of pull calls return a non-null uid within 60 days of first partner go-live | Fastly log: pull call outcomes per partner | -| Identity graph fill rate | >50% of EC hashes with at least 1 resolved partner ID within 60 days of partner go-live | KV scan sample | diff --git a/docs/internal/ssc_technical_spec.md b/docs/internal/ssc_technical_spec.md deleted file mode 100644 index 94028a03..00000000 --- a/docs/internal/ssc_technical_spec.md +++ /dev/null @@ -1,2165 +0,0 @@ -# Technical Specification: Edge Cookie (EC) - -**Status:** Draft -**Author:** Engineering -**PRD reference:** `docs/internal/ssc-prd.md` -**Last updated:** 2026-03-18 - ---- - -## Table of Contents - -1. [Overview](#1-overview) -2. [Architecture Overview](#2-architecture-overview) -3. [Module Structure](#3-module-structure) -4. [EC Identity Generation](#4-ec-identity-generation) -5. [Cookie and Header Handling](#5-cookie-and-header-handling) -6. [Consent Enforcement](#6-consent-enforcement) -7. [KV Store Identity Graph](#7-kv-store-identity-graph) -8. [Pixel Sync Endpoint (`GET /sync`)](#8-pixel-sync-endpoint-get-sync) -9. [S2S Batch Sync API (`POST /api/v1/sync`)](#9-s2s-batch-sync-api-post-apiv1sync) -10. [S2S Pull Sync (TS-Initiated)](#10-s2s-pull-sync-ts-initiated) -11. [Identity Resolution Endpoint (`GET /identify`)](#11-identity-resolution-endpoint-get-identify) -12. [Bidstream Decoration (`/auction` Mode B)](#12-bidstream-decoration-auction-mode-b) -13. [Partner Registry and Admin Endpoint](#13-partner-registry-and-admin-endpoint) -14. [Configuration](#14-configuration) -15. [Constants and Header Names](#15-constants-and-header-names) -16. [Error Handling](#16-error-handling) -17. [Request Routing](#17-request-routing) -18. [Testing Strategy](#18-testing-strategy) -19. [Implementation Order](#19-implementation-order) - ---- - -## 1. Overview - -Edge Cookie (EC) replaces SyntheticID as the primary user identity mechanism in Trusted Server. It uses a simpler, more stable signal (IP address + publisher passphrase), adds consent enforcement, and backs identity with a server-side KV graph that accumulates partner IDs over time. - -EC is the full replacement for SyntheticID. The PRD explicitly states backward compatibility is a non-goal. There is no coexistence, no fallback, no transitional period. - -**Prerequisites (must be merged before this epic begins):** - -- **SyntheticID removal** — [PR #479](https://github.com/IABTechLab/trusted-server/pull/479) removes SyntheticID from all active code paths: `get_or_generate_synthetic_id()`, `COOKIE_SYNTHETIC_ID`, `X-Synthetic-*` headers, `synthetic.rs` module, `settings.synthetic` config, and all SyntheticID generation/cookie code from `publisher.rs`, `endpoints.rs`, and `registry.rs`. It also renames `ConsentPipelineInput.synthetic_id` to `identity_key`, updates consent KV helper parameters/docs, and handles consent-store key migration (old SyntheticID keys orphaned, TTL expiry cleans them up). **This PR must be merged before implementation of this spec begins.** The spec assumes a codebase where SyntheticID no longer exists. Verify before starting: - - `grep -r 'synthetic_id' crates/` returns no hits outside test fixtures - - `grep -r 'X-Synthetic' crates/` returns no hits - - `trusted-server.toml` has no `[synthetic]` section - - `ConsentPipelineInput` uses `identity_key`, not `synthetic_id` -- **Consent implementation** — The consent pipeline (`build_consent_context()`, `ConsentContext`, `allows_ec_creation()`, TCF/GPP/US-Privacy decoding) is implemented and available as a stable interface before this epic. PR `#380` merged to `main`. EC calls `allows_ec_creation()` directly — no new gating functions are introduced. Note: EC changes the _phase order_ relative to the old SyntheticID flow — consent is evaluated before EC generation, so first-visit consent KV persistence is deferred to the second request (see §6.1.1 for full analysis). - -**Deferred from this spec (not in scope):** - -- TS Lite deployment mode (PRD Section 5) -- JOSE-signed KV entries / buyer attestation, and the associated `/.well-known/trusted-server.json` attestation object + `Cache-Control: max-age=3600` response (PRD Section 8.7). The existing discovery endpoint and its tests (`endpoints.rs:579–594`) assert only `version` and `jwks` fields — this spec does not modify that endpoint. Any addition of the PRD-required `attestation` field is deferred to when JOSE signing ships. -- Data deletion framework JWT endpoint (PRD Section 7.4) — the formal IAB-compliant deletion endpoint is deferred. The PRD explicitly acknowledges that manual KV deletion is the interim process until the formal endpoint ships, and states that regulated onboarding requires the formal endpoint to be in place first. This spec implements the manual-deletion-only interim; the JWT endpoint is a prerequisite for regulated onboarding and must be tracked separately. -- Winner notification EC headers on publisher ad server delivery — the current `/auction` path returns JSON inline to the JS caller; there is no server-to-server delivery step. A future delivery architecture is deferred. Note: §12.5 (auction response headers for the existing inline path) IS in scope; only the not-yet-built server-to-server delivery is deferred. - ---- - -## 2. Architecture Overview - -``` -Browser Request - │ - ▼ -┌─────────────────────────────────────────────────┐ -│ main.rs (router) │ -│ extract GeoInfo → enforce auth → route_request │ -└──────────┬──────────────────────────────────────┘ - │ -Two-phase model (matches existing codebase pattern): - -Phase 1 — pre-routing (like `GeoInfo::from_request()`): - ┌─────────────────────────────────────────┐ - │ EcContext::read_from_request() │ - │ - read ts-ec cookie / X-ts-ec header │ - │ - build_consent_context() → ConsentContext │ - │ - allows_ec_creation(consent) │ - │ No generation. No cookie writes. │ - └──────┬──────────────────────────────────┘ - │ -Phase 2 — inside organic handlers only: - ┌───────┼──────────────────────────────────────────────────┐ - │ │ │ - ▼ ▼ ▼ -handle_publisher_request() integration_registry.handle_proxy() -calls ec_context.generate_if_needed() calls ec_context.generate_if_needed() - -EC route handlers (GET /sync, GET /identify, POST /auction, -POST /api/v1/sync, POST /admin/*) NEVER call generate_if_needed(). -`/identify`, `/auction`, `POST /api/v1/sync`, and `POST /admin/*` -use `EcContext` in read-only form. `GET /sync` is the one exception: -it never bootstraps an EC, but it may replace `ec_context.consent` -with a locally-decoded fallback consent context for that request only -when the optional `consent` query param is the sole available signal. -/auction reads EC identity but never bootstraps it — the publisher -page-load path generates the EC before any auction request arrives. - -ec_finalize_response() — after every handler: - - consent withdrawn + cookie present? → clear_ec_on_response() + tombstone - - returning-user mismatch? → set_ec_on_response() [reconcile cookie to header EC] - - ec_generated == true? → set_ec_on_response() [new cookie only] -``` - -EC state flows through an `EcContext` struct created once per request and passed through handlers. - ---- - -## 3. Module Structure - -New files in `crates/common/src/`: - -``` -crates/common/src/ - ec/ - mod.rs — EcContext, pub re-exports - identity.rs — EC generation (HMAC-SHA256, IP normalization) - cookie.rs — create_ec_cookie(), delete_ec_cookie(), set_ec_on_response() - finalize.rs — ec_finalize_response() (cookie write/delete, last_seen, tombstone) - kv.rs — KvIdentityGraph, read/write/delete identity entries - partner.rs — PartnerRecord, PartnerStore, load_partner() - sync_pixel.rs — handle_sync() handler - sync_batch.rs — handle_batch_sync() handler - pull_sync.rs — PullSyncDispatcher, dispatch_background() - identify.rs — handle_identify() handler - admin.rs — handle_register_partner() handler -``` - -Existing files modified: - -| File | Change | -| -------------------------------- | ----------------------------------------------------- | -| `crates/common/src/settings.rs` | Add `EdgeCookie` settings struct | -| `crates/common/src/constants.rs` | Add EC header/cookie name constants | -| `crates/common/src/error.rs` | Add `EdgeCookie` error variant | -| `crates/common/src/auction/` | Inject EC into `user.id`, `user.eids`, `user.consent` | -| `crates/fastly/src/main.rs` | Register new routes, run EC middleware | - ---- - -## 4. EC Identity Generation - -### 4.1 Module: `ec/identity.rs` - -The EC generation mirrors the SyntheticID approach (`synthetic.rs`) but strips volatile inputs. - -```rust -/// Generates a fresh EC value from IP address and publisher passphrase. -/// -/// Output format: `{64-char hex HMAC-SHA256}.{6-char random alphanumeric}` -/// -/// # Errors -/// -/// Returns `EdgeCookie` error if HMAC computation fails. -pub fn generate_ec(passphrase: &str, ip: IpAddr) -> Result>; - -/// Normalizes an IP address for use as an HMAC input. -/// -/// - IPv4: returned as-is (`"203.0.113.1"`) -/// - IPv6: truncated to /64 prefix — first 4 hextets joined by `:`, lower-cased -/// (`"2001:db8:85a3:0"`) -/// - On dual-stack, the caller must supply the IPv6 address; this function does -/// not choose between them. -pub fn normalize_ip(ip: IpAddr) -> String; - -/// Extracts the stable 64-character hex prefix from a full EC value. -/// -/// The prefix is used as the KV store key. The `.suffix` is discarded. -/// -/// Returns `None` if the value is not in `{64-hex}.{6-alnum}` format. -pub fn ec_hash(ec_value: &str) -> Option<&str>; -``` - -**HMAC inputs (fixed — no template):** - -| Input | Value | -| --------- | ------------------------- | -| Message | `normalize_ip(client_ip)` | -| Key | `settings.ec.passphrase` | -| Algorithm | HMAC-SHA256 | - -**Output format:** `{64-char lowercase hex}.{6-char random alphanumeric}` - -The random suffix is generated with `fastly::rand` (same approach as SyntheticID). Once set in a cookie the full value is preserved; only the hash prefix is used as the KV key. - -**IPv6 /64 prefix:** Split on `:`, take first 4 groups, join with `:`. Example: -`2001:db8:85a3:0000:0000:8a2e:0370:7334` → `2001:db8:85a3:0`. - -**IP source:** Use `req.get_client_ip_addr()` — Fastly's trusted API that returns the verified client IP without relying on any request header. This is the same source used by the existing `synthetic.rs` IP handling. Do not fall back to `X-Forwarded-For` or any other header — those are forgeable by clients. If the API returns `None`, `EcContext.client_ip` is `None` and `generate_if_needed()` logs `warn` and skips EC generation — the page loads without an EC. This is best-effort; a missing client IP never produces a 500. - -On dual-stack: prefer IPv6 if the returned address is IPv6; otherwise use IPv4. - -### 4.2 EC Retrieval Priority - -Pre-routing, EC state is read (not generated) from the inbound request: - -1. `X-ts-ec` request header (forwarded by publisher infrastructure) -2. `ts-ec` cookie -3. Neither present → `ec_value = None`, `ec_was_present = false` - -When both header and cookie are present, the **header wins** as `ec_value` (used by handlers for identity reads, KV lookups, and response headers). `cookie_was_present` is still set to `true`. - -**Mismatch handling:** If the header and cookie carry different EC values, `EcContext` tracks both: - -- `ec_value` = header value (authoritative for handler reads) -- `cookie_ec_value` = cookie value (tracked separately for withdrawal) - -On consent **withdrawal** (`!allows_ec_creation && cookie_was_present`): - -- Delete the browser cookie (always, based on `cookie_was_present`) -- Tombstone the **cookie-derived** hash: `kv.write_withdrawal_tombstone(ec_hash(cookie_ec_value))` -- If the header-derived hash differs, also tombstone it: `kv.write_withdrawal_tombstone(ec_hash(ec_value))` -- This matches the existing SyntheticID behavior where revocation targets the cookie value (`publisher.rs:515`), not the header value. - -On **non-withdrawal** paths (last_seen, handler reads): use `ec_value` (header-derived) as the active identity. When `cookie_ec_value` is set (mismatch), `ec_finalize_response()` overwrites the browser cookie with the header-derived `ec_value` via `set_ec_on_response()`. This reconciles the browser identity to match the publisher-forwarded identity and prevents persistent oscillation between two ECs on subsequent requests. - -**Validation:** Both the header and cookie values are validated independently via `ec_hash()` (`{64-hex}.{6-alnum}` format). If the header is present but malformed, it is discarded and the cookie value is used instead (if valid). A malformed header must not suppress a valid cookie — bad forwarding infrastructure should not break returning-user identity. `cookie_was_present` is set based on the raw cookie existing, regardless of validity — an invalid cookie value is still a cookie that needs to be cleared on withdrawal. - -Generation (step 3 above becoming a new EC) happens only inside organic handlers — see §5.4. This logic lives in `EcContext::read_from_request()` (phase 1) and `EcContext::generate_if_needed()` (phase 2). - -### 4.3 `EcContext` - -```rust -/// Per-request Edge Cookie state. Constructed pre-routing once per request. -/// Organic handlers call `generate_if_needed()` to mint new ECs. `/sync` is the -/// one EC route that may replace `consent` with a locally-decoded fallback for -/// the remainder of that request only. -pub struct EcContext { - /// Full EC value (`hash.suffix`), if present on request or generated this request. - pub ec_value: Option, - /// Whether the `ts-ec` **cookie** was present on the inbound request. - /// This is the only field that gates consent-withdrawal cookie deletion — - /// the PRD's delete branch is conditioned on the cookie, not on X-ts-ec header. - pub cookie_was_present: bool, - /// The cookie's EC value, if different from `ec_value` (header won priority). - /// Used only for withdrawal: tombstone targets the cookie-derived hash to match - /// existing SyntheticID revocation behavior (`publisher.rs:515`). - /// `None` when cookie absent or cookie == header value. - pub cookie_ec_value: Option, - /// Whether any EC value was available (cookie OR X-ts-ec header). - pub ec_was_present: bool, - /// Set to true by `generate_if_needed()` when a new EC is minted this request. - /// `ec_finalize_response()` uses this to decide whether to write a Set-Cookie header. - pub ec_generated: bool, - /// Full consent context from the prerequisite consent pipeline. - /// Use `allows_ec_creation(&self.consent)` to derive a grant/deny decision. - /// Raw TCF/GPP strings (for KV writes and `user.consent`) are on `consent.raw_tc_string` - /// and `consent.raw_gpp_string`. - pub consent: ConsentContext, - /// Client IP extracted from `req` during `read_from_request()`. - /// Stored here so pull sync can use it after `req` has been consumed by routing. - /// `None` only if Fastly's `get_client_ip_addr()` returns `None`. - pub client_ip: Option, -} - -impl EcContext { - /// Phase 1: reads cookie/header and builds consent context. Does not generate. - /// Does not write to the **EC identity KV store**. Called pre-routing, like - /// `GeoInfo::from_request()` in the current `main.rs`. - /// - /// Calls `build_consent_context()` with the EC hash (when present) passed - /// via `ConsentPipelineInput.identity_key` (renamed from `synthetic_id` - /// in PR #479). - /// - /// When an EC hash is available (returning user), this enables the consent - /// pipeline's KV fallback (read) and KV persistence (write to the - /// **consent** KV store). On a first visit (no EC cookie), `ec_hash` is - /// `None` and no consent KV interaction occurs; consent is evaluated purely - /// from request cookies/headers. This means consent is not persisted to - /// consent KV until the user's second request. See §6.1.1. - pub fn read_from_request( - req: &Request, - settings: &Settings, - geo: Option<&GeoInfo>, - ) -> Result>; - - /// Phase 2: generates a new EC if none is present and consent is granted. - /// Called only inside organic handlers (`handle_publisher_request`, - /// `integration_registry.handle_proxy`). Never called by EC route handlers - /// or the auction handler — those consume EC identity but never bootstrap it. - /// Sets `ec_generated = true` when a new EC is minted, and writes the initial - /// KV entry via `kv.create_or_revive()` (best-effort — logs warn on failure, - /// does not block). Using `create_or_revive` (not `create`) ensures that a user - /// who re-consents within the 24-hour tombstone window recovers immediately. - /// - /// **Best-effort / never 500s organic traffic.** If EC generation fails - /// (e.g., `get_client_ip_addr()` returns `None`), the function logs `warn` - /// and returns without setting `ec_generated`. The organic handler proceeds - /// normally without an EC — the page still loads. Callers must NOT propagate - /// this error with `?`. - pub fn generate_if_needed( - &mut self, - settings: &Settings, - kv: &KvIdentityGraph, - ); - - /// Returns the stable 64-char hex prefix, or `None` if no EC. - pub fn ec_hash(&self) -> Option<&str>; -} -``` - -**`ec_finalize_response()` behavior** (signature: `ec_finalize_response(settings, geo, ec_context, kv, response)`): - -1. If `!allows_ec_creation(&consent) && cookie_was_present`: call `clear_ec_on_response()` (deletes cookie **and** strips any handler-built `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, and `X-ts-` response headers) and write withdrawal tombstones for each valid known EC hash (cookie-derived and, when different, header-derived). This runs on **every route** — consent withdrawal is always real-time enforced. Keyed on `cookie_was_present`, not `ec_was_present`, because only a cookie-held EC can be deleted by the browser. When the cookie is malformed and there is no valid header-derived hash, no tombstone is written. -2. If `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)`: call `kv.update_last_seen()` (debounced). If `cookie_ec_value.is_some()`, also call `set_ec_on_response()` to reconcile the browser cookie to the authoritative header-derived EC. -3. If `ec_generated == true`: call `set_ec_on_response()` — sets `Set-Cookie` and `X-ts-ec`. KV create already happened inside `generate_if_needed()`; `ec_finalize_response()` does NOT write KV beyond tombstones and `last_seen`. -4. Handler-built response headers (`X-ts-ec`, `X-ts-eids` set directly by `/identify`) are preserved on non-withdrawal paths only. - -**Note on `kv_degraded`:** Not on `EcContext` — `read_from_request()` does not read KV. Handlers track degraded state locally. `/identify` returns `degraded: true` in the JSON body on KV read failure; the auction handler treats a failed read as `eids: []`. - -```` - ---- - -## 5. Cookie and Header Handling - -### 5.1 Cookie attributes - -| Attribute | Value | -|-----------|-------| -| Name | `ts-ec` | -| Domain | `.{publisher.domain}` — derived by prepending `.` to `settings.publisher.domain`, **not** `settings.publisher.cookie_domain` | -| Path | `/` | -| Secure | Yes | -| SameSite | `Lax` | -| Max-Age | `31536000` (1 year) | -| HttpOnly | No | - -### 5.2 Module: `ec/cookie.rs` - -The `cookie_domain` parameter passed to all functions below is computed as -`format!(".{}", settings.publisher.domain)`. Do **not** use -`settings.publisher.cookie_domain` — that field is used by other cookie helpers -and does not carry the EC ownership guarantee. No startup validation change is -needed for `publisher.cookie_domain` — it continues to serve its existing -purpose for non-EC cookies. EC simply does not read it. - -```rust -/// Builds the `Set-Cookie` header value for a newly generated EC. -pub fn create_ec_cookie(ec_value: &str, cookie_domain: &str) -> String; - -/// Builds the `Set-Cookie` header value that expires (deletes) the EC cookie. -pub fn delete_ec_cookie(cookie_domain: &str) -> String; -// Sets Max-Age=0 with same Domain/Path/Secure/SameSite attributes. - -/// Sets the EC cookie and `X-ts-ec` response header on a response. -pub fn set_ec_on_response(response: &mut Response, ec_value: &str, cookie_domain: &str); - -/// Removes the EC cookie and strips all EC-related response headers: -/// `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, `x-ts-eids-truncated`, -/// and any `X-ts-` headers. Called on consent withdrawal to -/// prevent leaking EC identity in handler-built headers. -pub fn clear_ec_on_response(response: &mut Response, cookie_domain: &str); -```` - -### 5.3 Response header - -`X-ts-ec: {ec_hash.suffix}` is set by `set_ec_on_response()`, which is called by `ec_finalize_response()` in two cases: (1) `ec_generated == true` (new EC minted this request), or (2) `cookie_ec_value.is_some()` (header/cookie mismatch reconciliation — overwrites cookie to match header). It is also set explicitly by `/identify` and `/auction` handlers on their own response paths when an EC is present. It is **not** set on ordinary returning-user requests where the cookie already matches the header (or no header is present). - -This header is added to `INTERNAL_HEADERS` in `constants.rs` so it is stripped before proxying to downstream backends, consistent with existing `X-ts-*` handling. - -### 5.4 Per-request EC lifecycle - -**Phase 1 — pre-routing** (always runs, all routes): - -``` -EcContext::read_from_request() - Read ts-ec cookie value and X-ts-ec header value independently - Validate each via ec_hash() — returns None if not {64-hex}.{6-alnum} - If both valid: header wins as ec_value; cookie stored as cookie_ec_value (if differs) - If only header valid: ec_value = header, cookie_ec_value = None - If only cookie valid: ec_value = cookie, cookie_ec_value = None - If neither valid: ec_value = None - ec_was_present = ec_value.is_some() - cookie_was_present = ts-ec cookie raw key exists (regardless of validity) - ec_hash = ec_value.as_deref().and_then(ec_hash) // None on first visit or malformed - build_consent_context(jar, req, config, geo, ec_hash) → consent: ConsentContext - // ec_hash is the identity key for consent KV (renamed from synthetic_id in PR #479). - // When ec_hash is Some: consent KV fallback read + consent KV write (to consent store, not EC store). - // When ec_hash is None (first visit): no consent KV interaction — cookies/headers only. - ec_generated = false -``` - -**Phase 2 — inside organic handlers only** (`handle_publisher_request`, `handle_proxy`): - -``` -ec_context.generate_if_needed(settings, &kv) // best-effort — never 500s - └── allows_ec_creation(&consent) && ec_value == None? - → client_ip from ec_context.client_ip (captured in phase 1) - → client_ip is None? log warn, return (no EC generation possible) - → generate_ec(passphrase, ip) - → ec_value = Some(new_ec) - → ec_generated = true - → kv.create_or_revive(ec_hash, &entry) (best-effort, log warn if fails) - // create_or_revive overwrites a tombstone (ok=false) on re-consent - // no-ops if a live entry (ok=true) already exists -``` - -**`ec_finalize_response(settings, geo, ec_context, &kv, response)` — always runs, all routes:** - -``` - ├── !allows_ec_creation(&consent) && cookie_was_present? - │ → clear_ec_on_response() (delete cookie + strip ALL EC headers from response) - │ → // Tombstone all known valid EC hashes. May be 0, 1, or 2 hashes. - │ if let Some(cookie_hash) = cookie_ec_value.and_then(|v| ec_hash(&v)): - │ kv.write_withdrawal_tombstone(cookie_hash) // cookie-derived hash - │ if let Some(header_hash) = ec_value.and_then(|v| ec_hash(&v)): - │ if Some(header_hash) != cookie_hash: - │ kv.write_withdrawal_tombstone(header_hash) // header-derived hash (if different) - │ // When cookie is malformed and no valid header exists: no tombstone written. - │ // Cookie deletion is still the authoritative enforcement mechanism. - │ // Tombstone fails? log error, do NOT block — no retry possible on browser path. - │ - ├── ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)? - │ → kv.update_last_seen(ec_hash, now()) (returning user — debounced at 300s) - │ → if cookie_ec_value.is_some(): - │ // Header and cookie disagree — reconcile by overwriting cookie with header value. - │ // Prevents persistent split identity where user oscillates between two ECs - │ // depending on whether the forwarded header is present on subsequent requests. - │ set_ec_on_response() (Set-Cookie with ec_value, the header-derived identity) - │ - └── ec_generated == true? - → set_ec_on_response() (Set-Cookie + X-ts-ec on response) -``` - -EC route handlers (`GET /sync`, `GET /identify`, `POST /api/v1/sync`, `POST /admin/*`) never call `generate_if_needed()`. `ec_finalize_response()` will still delete the cookie on those routes if consent is withdrawn — that is intentional. - -**Cookie write rule:** `Set-Cookie` is written in exactly two cases: (1) `ec_generated == true` (first-time generation), or (2) `cookie_ec_value.is_some()` (header/cookie mismatch — reconcile cookie to match the header-derived identity). There is no cookie refresh or Max-Age reset on ordinary returning users where cookie already matches. The PRD defers a blanket refresh-on-every-request strategy to a future iteration. - ---- - -## 6. Consent Enforcement - -### 6.1 Prerequisite contracts - -Consent decoding shipped in `#380` (already merged). This spec treats the following as stable, pre-existing contracts — it does not implement them: - -- **`build_consent_context(input: &ConsentPipelineInput) -> ConsentContext`** — the main entry point. Extracts, decodes, and normalizes signals from cookies and headers. -- **`ConsentContext`** — carries: `raw_tc_string`, `raw_gpp_string`, `raw_us_privacy`, `gdpr_applies: bool`, `tcf: Option`, `gpp: Option`, `us_privacy: Option`, `expired: bool`, `gpc: bool`, `jurisdiction: Jurisdiction`, `source: ConsentSource` -- **`TcfConsent.has_storage_consent()`** — true when TCF Purpose 1 (store/access on device) is granted -- **`Jurisdiction { Gdpr, UsState(String), NonRegulated, Unknown }`** — detected privacy regime (from geo + config) -- **`UsPrivacy.opt_out_sale: PrivacyFlag`** — CCPA opt-out (`Yes`/`No`/`NotApplicable`) - -### 6.1.1 EC consent gating - -EC reuses the existing `allows_ec_creation(&ConsentContext) -> bool` function -from the consent module (`consent/mod.rs`). No parallel gating function is -introduced — EC calls `allows_ec_creation()` directly for all consent decisions -(EC generation, withdrawal detection, sync gating). - -There is no EC-specific consent gate and no behavior change to -`allows_ec_creation()` in this spec. Shared consent-policy semantics stay in -the consent module; EC only consumes that existing decision. - -**Consent pipeline integration:** - -`EcContext::read_from_request()` calls `build_consent_context()` with the EC hash as the identity key, passed via `ConsentPipelineInput.identity_key` (renamed from `synthetic_id` in PR #479). The consent pipeline's KV persistence and fallback behavior works with EC hashes: - -- **Returning user** (EC cookie present → `ec_hash` is `Some`): consent KV fallback read is available when consent cookies are absent; consent KV write persists cookie-sourced consent for future requests. Note: `build_consent_context()` calls `try_kv_write()` internally, so phase 1 writes to the **consent** KV store (not the EC identity store). -- **First visit** (no EC cookie → `ec_hash` is `None`): no consent KV interaction. Consent is evaluated purely from request cookies/headers. The gap: consent is not persisted to consent KV on the first request. This is accepted — in regulated jurisdictions (GDPR, US state), consent cookies/headers must be present for `allows_ec_creation()` to return `true`, so there is always a signal to persist on the next request. In non-regulated jurisdictions, `allows_ec_creation()` returns `true` without consent signals, so there is nothing to persist anyway. Consent KV persistence begins on the second request when the EC cookie is present. - -**Consent store keying:** Old consent KV entries under SyntheticID keys become orphaned after PR #479 ships. New entries are keyed by EC hash. Orphaned entries expire via TTL — no explicit migration is performed. - -**Rollout impact:** At cutover, returning users who relied on consent KV fallback (consent cookies absent, consent loaded from KV under SyntheticID key) will lose that fallback until a new EC-keyed consent entry is written on a subsequent request where consent cookies are present. This is a one-time window: once the EC cookie is set and a request with consent cookies arrives, the consent KV entry is written under the EC hash and fallback works again. The window duration depends on how quickly users return with consent cookies. This is accepted — consent cookies are the primary signal; KV fallback is a secondary mechanism for when cookies are blocked or absent. - -All downstream EC logic calls `allows_ec_creation(&self.consent)`. No consent decoding or gating logic is added in this epic. - -### 6.2 Consent withdrawal — KV delete - -When `allows_ec_creation(&consent)` returns `false` for a user whose **`ts-ec` cookie** is present (`cookie_was_present == true`). A user identified only by the `X-ts-ec` request header is not subject to cookie deletion — there is no cookie to expire. - -1. Issue `Set-Cookie: ts-ec=; Max-Age=0; ...` and strip all EC response headers (synchronous — must not fail silently). This always happens when `cookie_was_present == true`. -2. Write tombstone for each valid EC hash available (`cookie_ec_value` and/or `ec_value`). When neither is valid (malformed cookie, no header), **no tombstone is written** — cookie deletion alone is the enforcement mechanism. When at least one valid hash exists: `kv.write_withdrawal_tombstone(hash)` sets `consent.ok = false`, clears partner IDs, TTL 24h — approximately 25ms per write. - -The tombstone write runs in the request path (not async) to ensure real-time enforcement. Using a tombstone rather than a hard delete preserves the `consent_withdrawn` signal for batch sync clients for 24 hours — otherwise batch sync cannot distinguish consent withdrawal from an EC that never existed. - -If the tombstone write fails: - -- Log at `error` level with EC hash -- Do not block the response — cookie deletion is the primary enforcement mechanism -- **No retry is possible on the browser path.** Once the cookie is deleted, subsequent browser requests carry no EC value (`ec_hash()` returns `None`), so there is no hash to tombstone. A failed tombstone means batch sync clients may see `ec_hash_not_found` (after TTL expiry) rather than `consent_withdrawn` — this is accepted degradation. The cookie deletion remains the authoritative enforcement mechanism. - ---- - -## 7. KV Store Identity Graph - -### 7.1 Module: `ec/kv.rs` - -Two KV stores are used. Their names are configured in `trusted-server.toml`: - -| Store | TOML key | Purpose | -| ---------------- | ------------------ | ---------------------------------- | -| Identity graph | `ec.ec_store` | EC hash → identity JSON | -| Partner registry | `ec.partner_store` | Partner ID → config + API key hash | - -### 7.2 Identity graph schema - -**KV key:** 64-character hex hash (the stable prefix from `ec_value`, without `.suffix`). - -**KV value (JSON, max ~5KB):** - -```json -{ - "v": 1, - "created": 1741824000, - "last_seen": 1741910400, - "consent": { - "tcf": "CP...", - "gpp": "DBA...", - "ok": true, - "updated": 1741910400 - }, - "geo": { - "country": "US", - "region": "CA" - }, - "ids": { - "ssp_x": { "uid": "abc123", "synced": 1741824000 }, - "liveramp": { "uid": "LR_xyz", "synced": 1741890000 } - } -} -``` - -**KV metadata (max 2048 bytes, readable without streaming body):** - -```json -{ "ok": true, "country": "US", "v": 1 } -``` - -The `ok` field in metadata is a **historical consent record for S2S consumers only** — it is set to `false` by `write_withdrawal_tombstone()` so that batch sync clients (`POST /api/v1/sync`) can return `consent_withdrawn` rather than `ec_hash_not_found` during the 24-hour tombstone TTL. - -**`consent.ok` is NOT used to make the withdrawal decision on the main request path.** Consent withdrawal is determined entirely from `allows_ec_creation(&ec_context.consent)` on the current request. When withdrawal is detected, the cookie is deleted and `write_withdrawal_tombstone()` is called in-path (setting `ok = false`, 24h TTL — see §6.2). Engineers must not add a KV read to the consent withdrawal hot path based on this field. - -**Rust types:** - -```rust -pub struct KvEntry { - pub v: u8, - pub created: u64, - pub last_seen: u64, - pub consent: KvConsent, - pub geo: KvGeo, - pub ids: HashMap, -} - -pub struct KvConsent { - pub tcf: Option, - pub gpp: Option, - pub ok: bool, - pub updated: u64, -} - -pub struct KvGeo { - pub country: String, - pub region: Option, -} - -pub struct KvPartnerId { - pub uid: String, - pub synced: u64, -} - -pub struct KvMetadata { - pub ok: bool, - pub country: String, - pub v: u8, -} -``` - -### 7.3 TTL - -All KV writes use `time_to_live_sec = 31536000` (1 year), matching the cookie `Max-Age`. - -### 7.4 Conflict resolution — atomic read-modify-write - -Concurrent writes from different partners must not overwrite each other. Each partner's ID is namespaced under `ids[partner_id]` — a write for `ssp_x` must not clobber an existing `liveramp` entry. - -Implementation uses Fastly KV Store's **generation markers** (optimistic concurrency): - -```rust -pub struct KvIdentityGraph { - store_name: String, -} - -impl KvIdentityGraph { - pub fn new(store_name: impl Into) -> Self; - - /// Reads the full entry, returning the generation marker for CAS writes. - pub fn get( - &self, - ec_hash: &str, - ) -> Result, Report>; - - /// Reads only the metadata fields (consent flag, country). - pub fn get_metadata( - &self, - ec_hash: &str, - ) -> Result, Report>; - - /// Creates a new entry. Returns `Ok(())` if successful, `Err` if the key - /// already exists (concurrent create) or on KV error. - pub fn create( - &self, - ec_hash: &str, - entry: &KvEntry, - ) -> Result<(), Report>; - - /// Creates a new entry, OR overwrites an existing tombstone (`consent.ok = false`) - /// with a fresh entry when the user re-consents within the tombstone TTL. - /// - /// Behavior: - /// - No existing key → behaves identically to `create()`. - /// - Existing key with `consent.ok = false` (tombstone) → overwrites with - /// the new entry via CAS. Retries up to `MAX_CAS_RETRIES` on conflict. - /// - Existing key with `consent.ok = true` (live entry) → no-op, returns `Ok(())`. - /// - /// Called by `generate_if_needed()` instead of `create()`. This ensures that - /// re-consent recovery is immediate — a user who withdraws and then re-consents - /// within the 24-hour tombstone window gets a fresh identity entry without delay. - pub fn create_or_revive( - &self, - ec_hash: &str, - entry: &KvEntry, - ) -> Result<(), Report>; - - /// Atomically merges `ids[partner_id]` into the existing entry using a - /// generation marker. Retries up to `MAX_CAS_RETRIES` (3) times on - /// generation conflict before returning `Err`. - /// - /// If the key does not exist, creates a minimal live entry first: - /// `consent.ok = true`, `consent.tcf = None`, `consent.gpp = None`, - /// `created = synced`, `last_seen = synced`, `geo.country = "ZZ"`, - /// `geo.region = None`, and `ids = { partner_id: ... }`. - /// - /// This recovery path is intentional: it materializes the graph later when - /// the initial best-effort `create_or_revive()` on EC generation failed. - /// Batch sync still performs its explicit existence/tombstone check before - /// calling this method, so `POST /api/v1/sync` retains its `ec_hash_not_found` - /// contract. - pub fn upsert_partner_id( - &self, - ec_hash: &str, - partner_id: &str, - uid: &str, - synced: u64, - ) -> Result<(), Report>; - - /// Updates `last_seen` timestamp, but only if the stored value is more than - /// 300 seconds older than `timestamp`. This debounce prevents KV write - /// thrashing under bursty traffic — Fastly KV enforces a 1 write/sec limit - /// per key. Callers should log `warn` on failure and continue. - pub fn update_last_seen( - &self, - ec_hash: &str, - timestamp: u64, - ) -> Result<(), Report>; - - /// Writes a withdrawal tombstone for consent enforcement. - /// - /// Instead of hard-deleting the KV entry, this overwrites it with - /// `consent.ok = false`, clears all partner IDs, and sets a 24-hour TTL. - /// The tombstone allows batch sync clients (`POST /api/v1/sync`) to return - /// `consent_withdrawn` rather than `ec_hash_not_found` for the tombstone TTL. - /// - /// After the 24-hour TTL expires, the entry is gone. Any subsequent `get()` - /// returns `None` (`ec_hash_not_found`) — the distinction is time-bounded. - /// - /// Caller must handle `Err` by logging at `error` level; the cookie deletion - /// in `ec_finalize_response()` is the primary enforcement mechanism. - pub fn write_withdrawal_tombstone( - &self, - ec_hash: &str, - ) -> Result<(), Report>; - - /// Hard-deletes the entry. Used only for data deletion requests (IAB deletion - /// framework — deferred). For consent withdrawal, use `write_withdrawal_tombstone()`. - pub fn delete(&self, ec_hash: &str) -> Result<(), Report>; -} -``` - -`MAX_CAS_RETRIES = 3`. If all retries fail on a generation conflict, return `Err` — callers handle per-endpoint policy (§8.3 step 7 for pixel sync, §9.4 for batch sync). - -### 7.5 KV degraded behavior - -| Operation | KV unavailable | Action | -| ---------------------------------- | -------------- | ---------------------------------------------------------------------------------------------- | -| EC cookie creation | KV error | Set cookie. Skip KV create. Log `warn`. | -| `/sync` KV write | KV error | Redirect with `ts_synced=0&ts_reason=write_failed`. | -| `/identify` KV read | KV error | Return `200` with `ec` set, `degraded: true`, empty `uids`/`eids`. | -| `POST /api/v1/sync` | KV error | Return `207` with all mappings rejected, `reason: "kv_unavailable"`. | -| Pull sync KV write | KV error | Discard uid. Log `warn`. Retry on next qualifying request. | -| Consent withdrawal tombstone write | KV error | Delete cookie (primary enforcement). Log `error`. Next request: no cookie → no EC regenerated. | - ---- - -## 8. Pixel Sync Endpoint (`GET /sync`) - -### 8.1 Module: `ec/sync_pixel.rs` - -```rust -pub async fn handle_sync( - settings: &Settings, - kv: &KvIdentityGraph, - partner_store: &PartnerStore, - req: &Request, - ec_context: &mut EcContext, -) -> Result>; -``` - -### 8.2 Query parameters - -| Parameter | Required | Description | -| --------- | -------- | ---------------------------------------------------------------------------- | -| `partner` | Yes | Partner ID — must exist in `partner_store` | -| `uid` | Yes | Partner's user ID for this user | -| `return` | Yes | Redirect-back URL (must match partner's `allowed_return_domains`) | -| `consent` | No | Fallback TCF/GPP string if `ec_context.consent.is_empty()` after pre-routing | - -### 8.3 Flow - -``` -1. Parse query params. Missing required params → 400. - -2. Require a valid cookie-held EC. - If `cookie_was_present == false` OR `ec_context.ec_hash().is_none()` - (cookie missing or malformed) → redirect to - {return}?ts_synced=0&ts_reason=no_ec - -3. Look up partner record in partner_store. - Not found → 400. - -4. Validate return URL host against partner.allowed_return_domains. - - Exact hostname match only — no suffix or wildcard. - - Mismatch → 400. - -5. Evaluate consent. Use `ec_context.consent` (built pre-routing via - `build_consent_context()`). The optional `consent` query param is a **fallback - only** — used solely when `ec_context.consent.is_empty()` returns `true`. - This is the actual contract from the consent module. It is broader than - “no cookies or headers on the wire”: if consent KV fallback, decoded objects, - GPP section IDs, AC string, raw US privacy, or GPC already populated the - context, `is_empty()` is `false` and the query param is ignored entirely. - - When the fallback applies: decode the query param into a **locally-built** - `ConsentContext` (same TCF/GPP/USP decoders, same jurisdiction inputs), then - assign that value into `ec_context.consent` for the remainder of this request. - This makes the sync write decision and `ec_finalize_response()` use the same - effective consent view, avoiding a same-request “write partner ID, then - withdraw EC” conflict. Do NOT re-call `build_consent_context()` — that would - trigger `try_kv_write()` and persist the query-param consent to the consent KV - store, which is not intended. The decoded fallback applies only to this `/sync` - request; it is not written to the consent KV store and does not change any - future request unless the client sends real consent cookies/headers again. - - `!allows_ec_creation(...)` → redirect to {return}?ts_synced=0&ts_reason=no_consent - -6. Check anti-stuffing rate limit (sync_rate_limit per EC hash per partner per hour). - Exceeded → `429 Too Many Requests` (no redirect — the `return` URL is never called). - -7. kv.upsert_partner_id(ec_hash, partner_id, uid, now()) - If the root KV entry is missing (e.g. initial `create_or_revive()` failed on - the organic page load), `upsert_partner_id()` creates a minimal live entry and - then writes `ids[partner_id]`. This is the recovery path for best-effort EC - creation misses. - KV write failure → redirect to {return}?ts_synced=0&ts_reason=write_failed - -8. Success → redirect to {return}?ts_synced=1 -``` - -`ts_synced` values: - -| Value | Meaning | -| ------------------------------------ | ----------------------------- | -| `ts_synced=1` | KV write succeeded | -| `ts_synced=0&ts_reason=no_ec` | No valid EC cookie present | -| `ts_synced=0&ts_reason=no_consent` | Consent absent or denied | -| `ts_synced=0&ts_reason=write_failed` | KV write failed after retries | - -Rate limit exceeded returns `429 Too Many Requests` directly — the partner's `return` URL is not called in this case. - -### 8.4 Return URL construction - -Append `ts_synced` (and optional `ts_reason`) to the `return` URL: - -- If the URL already has a query string, append `&ts_synced=...` -- If not, append `?ts_synced=...` - -Do not modify any other query parameters on the `return` URL. - -### 8.5 Security - -- `return` URL validated by exact hostname match against `partner.allowed_return_domains`. No subdomain wildcard matching. -- No HMAC signature required on inbound sync request. -- Rate limit: `partner.sync_rate_limit` writes per EC hash per partner per hour. Default: 100. Configurable per partner in `partner_store`. - ---- - -## 9. S2S Batch Sync API (`POST /api/v1/sync`) - -### 9.1 Module: `ec/sync_batch.rs` - -```rust -pub async fn handle_batch_sync( - settings: &Settings, - kv: &KvIdentityGraph, - partner_store: &PartnerStore, - req: Request, -) -> Result>; -``` - -### 9.2 Authentication - -`Authorization: Bearer ` header required. Auth flow: - -1. Compute `sha256_hex(api_key)`. -2. Look up `partner_store.find_by_api_key_hash(hash)` — uses the `apikey:{hash}` secondary index (§13.1) for O(1) lookup instead of scanning all partners. -3. If the index returns a partner, verify the partner's stored `api_key_hash` matches the computed hash (constant-time comparison). This guards against stale index entries from key rotation. -4. If no match or verification fails → `401 Unauthorized` with no body processing. -5. If KV lookup fails (store unavailable) → `503 Service Unavailable`. - -Key rotation does not require binary redeployment — partners update via `/admin/partners/register`, which handles old API-key index cleanup (§13.1). - -### 9.2.1 API-key rate limiting - -After successful auth, check the API-key level rate limit: `partner.batch_rate_limit` requests per partner per minute (default 60). Uses the same Fastly rate-limiting API as pixel sync (§14.3), with key `batch:{partner_id}`. - -Exceeded → `429 Too Many Requests` with body `{ "error": "rate_limit_exceeded" }`. No mappings are processed. - -### 9.3 Request format - -``` -POST /api/v1/sync -Content-Type: application/json -Authorization: Bearer - -{ - "mappings": [ - { - "ec_hash": "<64-character hex hash>", - "partner_uid": "abc123", - "timestamp": 1741824000 - } - ] -} -``` - -Maximum batch size: 1000 mappings. Requests exceeding this receive `400 Bad Request`. - -### 9.4 Processing - -The authenticated partner's ID (from the `PartnerRecord` resolved via API key in §9.2) determines the `ids[partner_id]` namespace for all writes in this batch. A partner can only write to their own namespace. - -For each mapping: - -1. Validate `ec_hash` format (must be exactly 64 lowercase hex characters). Invalid format → reject with `reason: "invalid_ec_hash"`. -2. Read KV metadata for `ec_hash`. If not found → reject with `reason: "ec_hash_not_found"`. If `consent.ok = false` → reject with `reason: "consent_withdrawn"`. -3. `kv.upsert_partner_id(ec_hash, partner_id, partner_uid, timestamp)`. The upsert internally skips the write if the existing `ids[partner_id].synced ≥ timestamp` (idempotent — counted as accepted, no error). On KV failure → reject all remaining mappings with `reason: "kv_unavailable"`, return `207`. - -### 9.5 Response format - -```json -{ - "accepted": 998, - "rejected": 2, - "errors": [ - { "index": 45, "reason": "ec_hash_not_found" }, - { "index": 72, "reason": "consent_withdrawn" } - ] -} -``` - -HTTP status rules: - -| Condition | Status | -| -------------------------------------- | ------------------------------------------------------ | -| All mappings accepted | `200 OK` | -| Some accepted, some rejected | `207 Multi-Status` | -| All rejected (auth valid, batch valid) | `207 Multi-Status` with `accepted: 0` | -| Auth invalid | `401 Unauthorized` | -| Auth KV lookup failed (store down) | `503 Service Unavailable` | -| Malformed JSON or > 1000 mappings | `400 Bad Request` | -| KV entirely unavailable | `207 Multi-Status`, all rejected with `kv_unavailable` | - -```rust -pub struct BatchSyncResponse { - pub accepted: usize, - pub rejected: usize, - pub errors: Vec, -} - -pub struct BatchSyncError { - pub index: usize, - pub reason: BatchSyncRejection, -} - -#[derive(Debug, derive_more::Display)] -pub enum BatchSyncRejection { - #[display("invalid_ec_hash")] - InvalidEcHash, - #[display("ec_hash_not_found")] - EcHashNotFound, - #[display("consent_withdrawn")] - ConsentWithdrawn, - #[display("kv_unavailable")] - KvUnavailable, -} -``` - ---- - -## 10. S2S Pull Sync (TS-Initiated) - -### 10.1 Module: `ec/pull_sync.rs` - -Pull sync inverts the batch model: TS calls the partner's resolution endpoint server-to-server and writes the returned UID into the KV graph. No browser redirect is involved. - -```rust -pub struct PullSyncDispatcher { - concurrency_limit: usize, -} - -impl PullSyncDispatcher { - pub fn new(concurrency_limit: usize) -> Self; - - /// Dispatches pull sync calls for all qualifying partners. - /// Called after `send_to_client()` — fires outbound requests using - /// `Request::send_async()` which returns `PendingRequest` handles. - /// Internally: fires up to `concurrency_limit` requests via `send_async()`, - /// then calls `PendingRequest::wait()` (blocking) on each handle to collect - /// responses and write results to KV. This is synchronous blocking code - /// running after the client response is already flushed — no async runtime - /// needed. The Fastly WASM invocation remains alive until this returns. - pub fn dispatch_background( - &self, - ec_context: &EcContext, - client_ip: IpAddr, - partners: &[PartnerRecord], - kv: &KvIdentityGraph, - ); -} - -/// Fires a single partner pull request via `send_async()`, waits for the -/// response via `PendingRequest::wait()`, and writes the result to KV. -fn pull_one_partner( - ec_hash: &str, - ip: IpAddr, - partner: &PartnerRecord, - kv: &KvIdentityGraph, -); -``` - -### 10.2 Trigger conditions - -A pull sync is dispatched for a partner when all of the following are true on a request: - -1. The request was routed to an **organic handler** (`handle_publisher_request` or `integration_registry.handle_proxy`). Pull sync never fires on EC route handlers (`/sync`, `/identify`, `/api/v1/sync`, `/admin/*`) or `/auction`. This matches the PRD requirement that pull calls must not happen during the pixel sync flow. -2. A valid EC is present (`ec_context.ec_hash().is_some()`). This includes an EC - newly generated on the current organic request — pull sync may run immediately - after first-page EC creation because the response cookie is flushed before the - background dispatch starts. -3. `allows_ec_creation(&ec_context.consent) == true` -4. `partner.pull_sync_enabled == true` -5. Either: no entry exists for this partner in the KV graph, or the existing `synced` timestamp is older than `partner.pull_sync_ttl_sec` (default 86400 seconds) -6. Rate limit not exceeded: `partner.pull_sync_rate_limit` calls per EC hash per partner per hour (default 10) - -### 10.3 Execution model - -Pull calls are dispatched using Fastly's background task / `send_async` model after the response is flushed. They do not add latency to the user-facing request. - -Maximum concurrent pull calls per request: `settings.ec.pull_sync_concurrency` (default 3). - -**Architectural divergence from PRD:** The PRD describes excess partner calls being queued and dispatched on subsequent requests for the same user. A persistent queue is not implementable in the stateless Fastly WASM edge environment — there is no cross-request mutable state. This spec adapts the intent using a stateless rotating offset: sort qualifying partners by ID, then use `(unix_timestamp_secs / 3600) % partner_count` as the starting index (wrapping). This ensures different partners are prioritized across different requests without persisted state. Partners not called on a given request remain eligible on the next qualifying request per their `pull_sync_ttl_sec` condition. The practical outcome (all partners eventually called) matches the PRD intent; the mechanism differs due to the platform constraint. - -### 10.4 Outbound request - -``` -GET {partner.pull_sync_url}?ec_hash={64-char-hex}&ip={ip_address} -Authorization: Bearer {partner.ts_pull_token} -``` - -Before dispatching, `pull_sync.rs` validates that `pull_sync_url`'s hostname is present in `partner.pull_sync_allowed_domains`. If not, the call is skipped and an `error` is logged — this is a configuration error that should not occur at runtime if admin validation is working correctly (§13.2 step 3). - -Only the EC hash and IP are sent. No consent strings, geo data, or other partner IDs are included. - -**Expected partner responses:** - -```json -{ "uid": "abc123" } // resolved -{ "uid": null } // not recognized -``` - -Or `404 Not Found`. Both null and 404 are no-ops — no KV write, no error logged above `debug`. - -Any other non-200 response is treated as a transient failure. No retry. The next qualifying request triggers a new attempt. - -### 10.5 KV write on success - -On a non-null `uid`: call `kv.upsert_partner_id(ec_hash, partner_id, uid, now())`. If the root entry is missing, the upsert creates a minimal live entry first (same recovery path as `/sync`). On KV failure: log `warn` and discard the result. Retry occurs on the next qualifying request. - -The write updates `ids[partner_id].synced` to the current timestamp, resetting the `pull_sync_ttl_sec` window. - ---- - -## 11. Identity Resolution Endpoint (`GET /identify`) - -### 11.1 Module: `ec/identify.rs` - -```rust -pub async fn handle_identify( - settings: &Settings, - kv: &KvIdentityGraph, - partner_store: &PartnerStore, - req: &Request, - ec_context: &EcContext, -) -> Result>; -``` - -### 11.2 Call patterns - -**Browser-direct:** The browser sends the request to `ec.publisher.com/identify`. Cookies and consent cookies are sent automatically (same-site). No special header forwarding required. - -**Server-side proxy (for use case 2):** The publisher's origin server must forward: - -| Header | Required | -| --------------------------------------------------------- | -------------------------------------- | -| `Cookie: ts-ec=` or `X-ts-ec: ` | Yes | -| `Cookie: euconsent-v2=` or `Cookie: __gpp=` | Yes for EU/UK/US users | -| `X-consent-advertising: ` | Optional — takes precedence if present | - -### 11.3 EC and consent handling - -`/identify` follows `EcContext` retrieval priority (Section 4.2). It does **not** -generate a new EC, and the handler itself does not write cookies. However, -`ec_finalize_response()` still runs after the handler: on consent withdrawal it -deletes the EC cookie, and on header/cookie mismatch it may reconcile the cookie -to the authoritative header-derived EC. - -Consent is evaluated using the same logic as Section 6. - -### 11.4 Response - -**`200 OK` — EC present, consent granted:** - -```json -{ - "ec": "a1b2c3...AbC123", - "consent": "ok", - "degraded": false, - "uids": { - "uid2": "A4A...", - "liveramp": "LR_xyz" - }, - "eids": [ - { "source": "uidapi.com", "uids": [{ "id": "A4A...", "atype": 3 }] }, - { "source": "liveramp.com", "uids": [{ "id": "LR_xyz", "atype": 3 }] } - ] -} -``` - -`uids` contains one key per partner with `bidstream_enabled: true` and a resolved UID in the KV graph. Partners with no resolved UID for this user are omitted. - -**`200 OK` — KV unavailable (degraded):** - -```json -{ - "ec": "a1b2c3...AbC123", - "consent": "ok", - "degraded": true, - "uids": {}, - "eids": [] -} -``` - -**`200 OK` — EC present, KV entry missing (no synced partners yet):** - -This case occurs by design when `create_or_revive()` fails on EC generation (best-effort) or when the EC was just created and no partners have synced yet. It is not an error — the EC is valid, just has no partner data. - -```json -{ - "ec": "a1b2c3...AbC123", - "consent": "ok", - "degraded": false, - "uids": {}, - "eids": [] -} -``` - -Note: `degraded` is `false` because the KV read succeeded (it returned `None`, meaning no entry exists). `degraded: true` is reserved for KV read errors where the entry might exist but couldn't be retrieved. - -**`403 Forbidden` — consent denied (regardless of EC presence):** - -```json -{ "consent": "denied" } -``` - -Consent is evaluated **before** EC presence. If `!allows_ec_creation(&consent)`, return `403` immediately — do not fall through to the `204` branch. This ensures consent denial is always surfaced, even for users with no EC. - -**`204 No Content` — no EC present, consent not denied.** No body. - -### 11.5 Response headers (supplementary) - -Set on `200` responses only: - -| Header | Value | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `X-ts-ec` | `{ec_hash.suffix}` | -| `X-ts-eids` | Standard base64 (RFC 4648, with `=` padding) of the JSON array of OpenRTB 2.6 `user.eids` objects. Capped at **4 KB** after encoding. If the encoded value exceeds 4 KB, the array is truncated (fewest partners first — highest `synced` timestamp retained) until it fits, and a `x-ts-eids-truncated: true` header is added. | -| `X-ts-` | Resolved UID per partner (e.g., `X-ts-uid2`). One header per partner with a resolved UID. **Capped at 20 partners** — partners sorted by most-recently synced; excess partners are omitted silently. | -| `X-ts-ec-consent` | `ok` (always — denied consent returns `403`, not `200`) | - -These are supplementary — callers should read the JSON body as the primary contract. The 4 KB cap on `X-ts-eids` and the 20-partner cap on `X-ts-` headers reflect typical proxy and browser total-header-budget constraints. Both caps apply independently. - -### 11.6 Performance target - -`/identify` must respond within 30ms (excluding network latency) when EC is present and KV read succeeds. This requires the KV read to be on the fast path with no retries. - -CORS headers must be set to allow browser-direct calls from the publisher's page. The `Access-Control-Allow-Origin` header is dynamically reflected from the `Origin` request header if the origin is an exact match or a subdomain of `settings.publisher.domain`: - -``` -// e.g. publisher.domain = "example.com" -// Allowed: https://example.com, https://www.example.com, https://news.example.com -// Rejected: https://evil.com, https://notexample.com - -Access-Control-Allow-Origin: -Access-Control-Allow-Credentials: true -Access-Control-Allow-Methods: GET, OPTIONS -Access-Control-Allow-Headers: Cookie, X-ts-ec, X-consent-advertising -Access-Control-Expose-Headers: X-ts-ec, X-ts-eids, X-ts-ec-consent, X-ts-eids-truncated, -Vary: Origin -``` - -**`Access-Control-Expose-Headers` note:** The dynamic `X-ts-` headers must be enumerated per-response, not as a static constant. The handler builds the expose list by iterating the partner IDs that have resolved UIDs in the response. `x-ts-eids-truncated` is always included in the expose list (browser JS should be able to detect truncation even when it occurs). - -**Origin validation logic:** CORS headers are only relevant when the `Origin` request header is present (browser requests always send it; server-side proxy calls typically do not). - -- **No `Origin` header present:** Process normally. No CORS headers added. No `403`. This is the server-side proxy path from §11.2 — origin-server calls forwarding `Cookie` and consent headers. -- **`Origin` header present, hostname matches `publisher.domain` or ends with `.{publisher.domain}` and scheme is `https`:** Reflect origin in `Access-Control-Allow-Origin`. Add `Vary: Origin`. -- **`Origin` header present but does not match:** Return `403`. No body. - -Browser `fetch()` with `credentials: "include"` sends an `OPTIONS` preflight. The router handles `OPTIONS /identify` identically — returns `200 OK` with the CORS headers above and no body. - ---- - -## 12. Bidstream Decoration (`/auction` Mode B) - -### 12.1 Changes to existing auction path - -The auction handler (`crates/common/src/auction/`) is modified to inject EC identity into outbound OpenRTB requests. This is **not** a builder tweak — it requires explicit schema additions across multiple files. SyntheticID is fully removed from the auction path — no fallback, no `X-Synthetic-*` headers, no `get_or_generate_synthetic_id()`. - -| Concern | Behavior | -| ------------------------------------- | ----------------------------------------------------------------------------------------------------------- | -| `UserInfo.id` | Replace with `ec_value` when EC is present. Remove `synthetic_id` field. When no EC → `user.id` is omitted. | -| Outbound OpenRTB `user.id` | Set to `ec_value` when EC present. Omit when no EC (no fallback). | -| `X-Synthetic-*` response headers | **Removed.** Replaced by `X-ts-ec`. | -| `X-ts-ec` response header | Set when EC is present. | -| Publisher and integration proxy paths | Only `ec_context.generate_if_needed()` runs. `get_or_generate_synthetic_id()` is removed. | -| `convert_tsjs_to_auction_request()` | Takes `ec_context: &EcContext` (not Optional). SyntheticID parameter removed. | - -**Schema changes required before handler changes:** - -| File | Change | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `types.rs` | Replace `id: String` in `UserInfo` with `ec_value: Option`. Add `Eid` and `EidUid` OpenRTB 2.6 types. Remove synthetic fields. | -| `openrtb.rs` | Add `eids: Vec` and `consent: Option` to `User` struct. Remove `ext.synthetic_fresh`. | -| `prebid.rs` | Populate `user.id` from EC value. Add `user.eids`, `user.consent`. Remove synthetic fallback. | -| `formats.rs` | Accept `ec_context: &EcContext` (not Optional). Remove `synthetic_id` parameter. | -| `endpoints.rs` | Remove `get_or_generate_synthetic_id()` call. Remove `X-Synthetic-*` headers. Use `ec_context.consent` instead of internal `build_consent_context()`. Pass `ec_context` to `convert_tsjs_to_auction_request()`. Add `X-ts-ec` header. | - -These changes affect the OpenRTB wire format — confirm with engineering that no existing SSP integrations break before merging. - -### 12.2 `user` object injection - -When an `EcContext` is available on the request, the auction handler performs an explicit KV read before building the OpenRTB request: - -```rust -// In handle_auction(): -let (user_id, eids) = match ec_context.ec_hash() { - Some(hash) => { - let kv_entry = kv.get(hash).ok().flatten(); - let eids = match kv_entry { - Some((entry, _gen)) => build_eids_from_kv(&entry, partner_store), - None => vec![], // KV read failed or no entry — degrade gracefully - }; - (ec_context.ec_value.clone(), eids) - } - None => (None, vec![]), // No EC — user.id omitted, no EIDs. Auction still runs. -}; - -user.id = user_id; -user.consent = consent_string; // TCF string from ec_context.consent, else None -user.eids = eids; -``` - -`build_eids_from_kv` iterates `kv_entry.ids` and includes only partners with `bidstream_enabled: true` and a non-empty `uid`. Partners without a resolved UID are omitted. - -### 12.3 OpenRTB `user.eids` structure - -```json -{ - "user": { - "id": "a1b2c3...AbC123", - "consent": "CP...", - "eids": [ - { - "source": "liveramp.com", - "uids": [{ "id": "LR_xyz", "atype": 3 }] - }, - { - "source": "uidapi.com", - "uids": [{ "id": "A4A...", "atype": 3 }] - } - ] - } -} -``` - -`atype: 3` for all EC-derived IDs (partner-defined), per OpenRTB 2.6 spec. - -### 12.4 SSP-specific adapter `ext.eids` - -When calling a specific PBS adapter, include only that SSP's resolved ID in the adapter-level `ext.eids`. The full `user.eids` array contains all configured identity providers. - -### 12.5 `/auction` response headers (in-scope) - -The current `/auction` path returns a JSON response inline to the JS caller (`endpoints.rs:71`). There is no server-to-server delivery step to a publisher ad server. EC headers are added to this existing response: - -| Header | Value | -| --------------------- | ------------------------------------------------------------------------------------------------------------------ | -| `X-ts-ec` | `{ec_hash.suffix}` — when EC is present | -| `X-ts-eids` | Standard base64 (RFC 4648) of OpenRTB 2.6 `user.eids` JSON array. Capped at 4 KB — same truncation rules as §11.5. | -| `X-ts-eids-truncated` | `true` — present only when `X-ts-eids` was truncated | -| `X-ts-ec-consent` | `ok` — only present when consent granted; on withdrawal `ec_finalize_response()` strips all EC headers | - -**Deferred:** A future server-to-server winner-notification delivery step to a publisher ad server is not in scope for this iteration. See §1 deferred items. - ---- - -## 13. Partner Registry and Admin Endpoint - -### 13.1 Module: `ec/partner.rs` - -```rust -pub struct PartnerRecord { - /// Partner identifier. Must match `^[a-z0-9_-]{1,32}$` (lowercase, no spaces). - /// Used to build `X-ts-` response headers — header-safety is required. - /// Reserved names that would collide with existing managed headers are rejected - /// at registration: `ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`. - pub id: String, - pub name: String, - pub allowed_return_domains: Vec, - pub api_key_hash: String, // SHA-256 hex of the partner's API key - pub bidstream_enabled: bool, - pub source_domain: String, // OpenRTB source (e.g., "liveramp.com") - pub openrtb_atype: u8, // typically 3 - pub sync_rate_limit: u32, // per EC hash per partner per hour - pub batch_rate_limit: u32, // API-key level: requests per partner per minute (default 60) - pub pull_sync_enabled: bool, - pub pull_sync_url: Option, // required when pull_sync_enabled; validated at registration - pub pull_sync_allowed_domains: Vec, // allowlist of domains TS may call for this partner - pub pull_sync_ttl_sec: u64, // default 86400 - pub pull_sync_rate_limit: u32, // default 10 - pub ts_pull_token: Option, // required when pull_sync_enabled; outbound bearer token -} - -pub struct PartnerStore { - store_name: String, -} - -impl PartnerStore { - pub fn new(store_name: impl Into) -> Self; - - /// Looks up a partner by ID. Returns `None` if not found. - pub fn get(&self, partner_id: &str) -> Result, Report>; - - /// Verifies an API key against the stored hash for a given partner. - /// Uses constant-time comparison. - pub fn verify_api_key(&self, partner_id: &str, api_key: &str) -> bool; - - /// Writes or updates a partner record. - /// Returns `true` if this was a new partner (create), `false` if an existing - /// partner was updated. The pre-read needed for index maintenance (old API key - /// deletion) also determines this. - pub fn upsert(&self, record: &PartnerRecord) -> Result>; - - /// Looks up the partner owning a given API key hash (for batch sync auth). - /// Uses the `apikey:{hash}` secondary index for O(1) lookup, then verifies the - /// stored `api_key_hash` matches (guards against stale index from key rotation). - pub fn find_by_api_key_hash(&self, hash: &str) -> Result, Report>; - - /// Returns all partner records with `pull_sync_enabled == true`. - /// Used by the pull sync dispatcher after each organic request. Implementations - /// must re-check `pull_sync_enabled` on the fetched record before returning it, - /// because the `_pull_enabled` secondary index is best-effort and may be stale. - pub fn pull_enabled_partners(&self) -> Result, Report>; -} -``` - -**Storage layout:** Partner records are stored as JSON values in `partner_store` KV, keyed by `partner_id`. Two operations require access patterns beyond single-key lookup: - -1. **`find_by_api_key_hash(hash)`** — batch sync auth needs to find the partner owning a given API key hash. Implementation: maintain a secondary index entry `apikey:{sha256_hex} → partner_id` in the same KV store. Written on `upsert()`, looked up on batch auth. **On key rotation:** `upsert()` must read the existing record first, and if the `api_key_hash` has changed, delete the old `apikey:{old_hash}` index entry before writing the new one. This prevents old API keys from remaining valid after rotation. - -2. **`pull_enabled_partners()`** — pull sync needs all partners with `pull_sync_enabled == true`. Implementation: maintain an index entry `_pull_enabled → [partner_id_1, partner_id_2, ...]` (JSON array of partner IDs) in the same KV store. Updated on `upsert()` when `pull_sync_enabled` changes. The dispatcher reads this list, then does individual `get()` calls for each partner record. This bounds the number of KV reads to `1 + pull_partner_count` per organic request. - -**Consistency model:** These index writes are **best-effort, not atomic** — Fastly KV does not support multi-key transactions. `upsert()` writes in order: (1) primary record, (2) old API-key index deletion (if key changed), (3) new API-key index, (4) `_pull_enabled` list. If the process fails mid-sequence, indexes may be stale. All readers handle this defensively: - -- `find_by_api_key_hash()`: if the index points to a partner whose stored `api_key_hash` does not match the lookup hash, treat as auth failure (stale index from a rotation). -- `pull_enabled_partners()`: if a listed partner ID returns `None` from `get()`, skip it silently. If the fetched record has `pull_sync_enabled == false`, also skip it silently — that is a stale `_pull_enabled` index entry. -- The `_pull_enabled` list is vulnerable to lost updates under concurrent registrations. This is accepted — partner registration is a low-frequency admin operation (not a hot path). If lost updates become an issue, a CAS-based read-modify-write can be added later. - -### 13.2 Admin endpoint (`POST /admin/partners/register`) - -**Module:** `ec/admin.rs` - -> **Codebase invariant — requires test update:** `Settings::ADMIN_ENDPOINTS` in `settings.rs` lists routes that must be covered by a `[[handlers]]` Basic Auth entry. The existing test at `settings.rs:1504-1530` scans `main.rs` for **every** `/admin/` route string and asserts it appears in `ADMIN_ENDPOINTS`. When `/admin/partners/register` is added to `main.rs`, this test will fail. -> -> **Required changes:** -> -> 1. Do **NOT** add `/admin/partners/register` to `ADMIN_ENDPOINTS` — it uses bearer-token-in-handler auth. -> 2. Update the admin-route-scan test (`settings.rs:1504-1530`) to maintain an exclusion list of bearer-token-authed admin routes (e.g., `const BEARER_AUTH_ADMIN_ROUTES: &[&str] = &["/admin/partners/register"]`) and skip those when asserting `ADMIN_ENDPOINTS` coverage. -> 3. Narrow the `[[handlers]]` pattern in `trusted-server.toml` from `"^/admin"` to `"^/admin/keys"` so that `/admin/partners/register` is not intercepted by `enforce_basic_auth()` before reaching its bearer-token handler. - -```rust -pub async fn handle_register_partner( - settings: &Settings, - partner_store: &PartnerStore, - req: Request, -) -> Result>; -``` - -Authentication: `Authorization: Bearer ` header, validated inside the handler against `settings.ec.admin_token_hash` (SHA-256 constant-time comparison). This is a publisher-level admin credential — separate from partner API keys, and enforced in-handler (not via `[[handlers]]` Basic Auth). Returns `401 Unauthorized` with no body if the token is missing or invalid. - -**Request:** - -``` -POST /admin/partners/register -Authorization: Bearer -Content-Type: application/json - -{ - "id": "ssp_x", - "name": "SSP Example", - "allowed_return_domains": ["sync.example-ssp.com"], - "api_key": "raw_key_to_hash_and_store", - "bidstream_enabled": true, - "source_domain": "example-ssp.com", - "openrtb_atype": 3, - "sync_rate_limit": 100, - "batch_rate_limit": 60, - "pull_sync_enabled": false, - "pull_sync_url": null, - "pull_sync_allowed_domains": [], - "pull_sync_ttl_sec": 86400, - "pull_sync_rate_limit": 10, - "ts_pull_token": null -} -``` - -**Processing:** - -1. Validate `Authorization: Bearer `: SHA-256 hash the token and compare against `settings.ec.admin_token_hash` using constant-time comparison. `401` if missing or invalid. -2. Validate required fields (`id`, `name`, `allowed_return_domains`, `api_key`, `source_domain`). `400` on failure. - Validate `id` format: must match `^[a-z0-9_-]{1,32}$`. Must not be a reserved name - (`ec`, `eids`, `ec-consent`, `eids-truncated`, `synthetic`, `ts`, `version`, `env`). `400` with descriptive message on failure. -3. If `pull_sync_enabled == true`, validate that both `pull_sync_url` and `ts_pull_token` are present and non-empty. `400` with `"pull_sync_url and ts_pull_token are required when pull_sync_enabled is true"` if either is missing. - If `pull_sync_url` is set, validate that its hostname is present in `pull_sync_allowed_domains`. `400` on failure with `"pull_sync_url domain must be in pull_sync_allowed_domains"`. This prevents TS from being directed to call arbitrary URLs — the allowlist must be declared in the same registration payload. -4. Hash `api_key` with SHA-256 before writing — never store plaintext. -5. `let created = partner_store.upsert(record)?`. `503` on KV failure. - `upsert()` returns `true` for a new partner, `false` for an update. -6. Return `201 Created` if new partner (`created == true`), or `200 OK` if update - (`created == false`). Use an explicit response DTO — do NOT serialize the full - `PartnerRecord` (which contains `api_key_hash` and `ts_pull_token`). - -**Response:** - -```json -{ - "id": "ssp_x", - "name": "SSP Example", - "pull_sync_enabled": false, - "bidstream_enabled": true, - "created": true -} -``` - -The response confirms the registration succeeded and echoes key fields. `api_key_hash`, `ts_pull_token`, and `api_key` are never returned. `PartnerRecord` does not have a `registered_at` field — use the `created` boolean to signal first registration vs. upsert update. - ---- - -## 14. Configuration - -### 14.1 New `EdgeCookie` settings struct - -Added to `crates/common/src/settings.rs`: - -```rust -#[derive(Debug, Clone, Deserialize, Serialize, Validate)] -pub struct EdgeCookie { - /// Publisher passphrase used as HMAC key for EC generation. - /// Must be identical across all of the publisher's owned domains. - /// Publishers sharing this value with partners form an identity-federated consortium. - #[validate(custom(function = EdgeCookie::validate_passphrase))] - pub passphrase: String, - - /// Fastly KV store name for the EC identity graph. - #[validate(length(min = 1))] - pub ec_store: String, - - /// Fastly KV store name for the partner registry. - #[validate(length(min = 1))] - pub partner_store: String, - - /// SHA-256 hex of the publisher admin token for `POST /admin/partners/register`. - /// The plaintext token is provided in the `Authorization: Bearer` header; - /// it is never stored in plaintext. - #[validate(custom(function = EdgeCookie::validate_sha256_hex))] - pub admin_token_hash: String, - - /// Maximum concurrent pull sync calls dispatched per request. - #[validate(range(min = 1))] - #[serde(default = "EdgeCookie::default_pull_sync_concurrency")] - pub pull_sync_concurrency: usize, -} - -impl EdgeCookie { - fn validate_passphrase(passphrase: &str) -> Result<(), ValidationError>; - // Rejects "passphrase" or empty string as placeholder. - - fn validate_sha256_hex(value: &str) -> Result<(), ValidationError>; - // Requires exactly 64 lowercase hex characters. - - fn default_pull_sync_concurrency() -> usize { 3 } -} -``` - -Added to `Settings`: - -```rust -pub struct Settings { - // ... existing fields ... - #[validate(nested)] - pub ec: EdgeCookie, // Required — omitting [ec] is a startup error -} -``` - -`EdgeCookie` does not derive `Default` — omitting the `[ec]` section from TOML is a deserialization error at startup. This is intentional: `passphrase`, `ec_store`, `partner_store`, and `admin_token_hash` have no safe defaults. The `#[validate(nested)]` attribute ensures `EdgeCookie::validate_passphrase()` runs when `settings.validate()` is called at startup (`settings_data.rs:28`), matching the pattern used by `Publisher` and `Rewrite` in the existing `Settings` struct (`Synthetic` is removed in PR #479). - -### 14.2 TOML configuration example - -```toml -[ec] -passphrase = "publisher-chosen-secret" -ec_store = "ec_identity_store" -partner_store = "ec_partner_store" -admin_token_hash = "sha256-hex-of-publisher-admin-token" -pull_sync_concurrency = 3 -``` - -### 14.3 Rate Limit Storage - -Pixel sync and pull sync rate limits (per EC hash per partner per hour) cannot use in-memory state in a WASM/Fastly Compute environment — there is no shared memory across requests. - -**Implementation:** Use Fastly's Edge Rate Limiting API (`fastly::erl::RateCounter`), which provides distributed per-key counting without KV latency and is designed for high-frequency counting without per-key write limits. - -| Counter | Key format | Window | -| ---------- | ----------------------------- | -------- | -| Pixel sync | `{partner_id}:{ec_hash}` | 1 hour | -| Pull sync | `pull:{partner_id}:{ec_hash}` | 1 hour | -| Batch sync | `batch:{partner_id}` | 1 minute | - -Engineering must confirm `fastly::erl::RateCounter` availability in the target before implementation of Steps 7, 9, and 10 is considered complete. Do NOT silently skip rate limiting in production if ERL is unavailable. Do NOT fall back to KV-based counters — they would hit the same 1 write/sec/key limit that necessitates `update_last_seen()` debouncing, and would thrash under real sync traffic. If ERL is unavailable, the rate-limited routes are blocked on an approved alternative counting mechanism. - -### 14.4 Deprecation note - -`settings.synthetic` is removed in PR #479. The `[synthetic]` TOML section, `counter_store`, `opid_store`, and `secret_key` fields are no longer present. - ---- - -## 15. Constants and Header Names - -New constants in `crates/common/src/constants.rs`: - -```rust -// EC cookie name -pub const COOKIE_EC: &str = "ts-ec"; - -// EC response header -pub const HEADER_X_TS_EC: &str = "x-ts-ec"; - -// Supplementary identity headers -pub const HEADER_X_TS_EIDS: &str = "x-ts-eids"; -pub const HEADER_X_TS_EC_CONSENT: &str = "x-ts-ec-consent"; -pub const HEADER_X_TS_EIDS_TRUNCATED: &str = "x-ts-eids-truncated"; - -// Consent cookies (must match existing constants in constants.rs) -pub const COOKIE_TCF: &str = "euconsent-v2"; -pub const COOKIE_GPP: &str = "__gpp"; -pub const COOKIE_GPP_SID: &str = "__gpp_sid"; -pub const COOKIE_US_PRIVACY: &str = "us_privacy"; - -// No EC-specific geo/IP header constants — use req.get_client_ip_addr() and GeoInfo::from_request(req). -``` - -The following EC headers must be added to `INTERNAL_HEADERS` in `constants.rs` to ensure they are stripped before proxying to downstream backends: - -- `HEADER_X_TS_EC` (`x-ts-ec`) -- `HEADER_X_TS_EIDS` (`x-ts-eids`) -- `HEADER_X_TS_EC_CONSENT` (`x-ts-ec-consent`) -- `HEADER_X_TS_EIDS_TRUNCATED` (`x-ts-eids-truncated`) -- Dynamic `X-ts-` headers — these cannot be registered statically because partners are added at runtime via `/admin/partners/register`. The `INTERNAL_HEADERS` filter **must use prefix stripping** (`x-ts-` prefix match) rather than enumerating partner IDs. A startup snapshot would miss partners registered after deployment. The current filter in `http_util.rs` uses explicit header names — extend it to also strip any header matching the `x-ts-` prefix pattern. - ---- - -## 16. Error Handling - -New error variants in `crates/common/src/error.rs`: - -```rust -pub enum TrustedServerError { - // ... existing variants ... - - /// Edge Cookie operation failed — used only for EC-specific route handler - /// errors (e.g., KV read failure in /identify). EC generation failure on - /// organic routes does NOT produce this error — it is best-effort (log warn, - /// continue without EC). Missing client IP is logged but never surfaced as 500. - #[display("Edge Cookie error: {message}")] - EdgeCookie { message: String }, - // Maps to StatusCode::INTERNAL_SERVER_ERROR (500) - // Used for: EC-specific handler errors only (not organic-path generation) - - /// Partner not found in partner_store. - #[display("Partner not found: {partner_id}")] - PartnerNotFound { partner_id: String }, - // Maps to StatusCode::BAD_REQUEST (400) - - /// Partner API key authentication failed. - #[display("Invalid API key for partner: {partner_id}")] - PartnerAuthFailed { partner_id: String }, - // Maps to StatusCode::UNAUTHORIZED (401) -} -``` - ---- - -## 17. Request Routing - -New routes added to `route_request()` in `crates/fastly/src/main.rs`: - -```rust -// EC sync pixel — no auth required (partner validation is internal) -(GET, "/sync") → handle_sync(settings, &kv, &partner_store, &req, &mut ec_context) - -// EC identity resolution — no auth required (consent-gated) -(GET, "/identify") → handle_identify(settings, &kv, &partner_store, &req, &ec_context) - -// CORS preflight for /identify — must be registered explicitly, current router dispatches by exact method/path -(OPTIONS, "/identify") → cors_preflight_identify(settings, &req) - -// S2S batch sync — partner API key auth (internal to handler) -(POST, "/api/v1/sync") → handle_batch_sync(settings, &kv, &partner_store, req) - -// Partner registration — publisher admin auth enforced in-handler (Bearer token) -(POST, "/admin/partners/register") → handle_register_partner(settings, &partner_store, req) -``` - -Route ordering: EC routes are inserted before the fallback `handle_publisher_request()`. The `/admin/partners/register` route uses bearer-token auth in-handler (not `[[handlers]]` Basic Auth). The current `trusted-server.toml` has `path = "^/admin"` which catches **all** `/admin/*` paths via `enforce_basic_auth()` before routing — this would block bearer-token requests to `/admin/partners/register`. **Required change:** narrow the existing `[[handlers]]` pattern from `"^/admin"` to `"^/admin/keys"` so it covers only `/admin/keys/rotate` and `/admin/keys/deactivate` (the routes in `Settings::ADMIN_ENDPOINTS`). `/admin/partners/register` then passes through `enforce_basic_auth()` unchallenged and reaches the bearer-token handler. - -### 17.1 EC integration in `main.rs` - -EC follows the same pre-routing pattern as `GeoInfo::from_request()` (line 70). The pull sync background step requires a **structural refactor of the Fastly entrypoint**: - -1. `route_request()` return type changes from `Result` to `Result<(), Error>`. -2. The response is flushed mid-function via `response.send_to_client()` instead of being returned to `main()`. -3. The `#[fastly::main]` function (`main.rs:32`) currently returns `Result` — it must change to call `route_request()` and return `Ok(())` (or map the error). The current `fn main(req: Request) -> Result` signature is incompatible with the `send_to_client()` pattern. -4. After `send_to_client()`, the WASM invocation continues for background pull sync work. - -This is a supported Fastly Compute pattern — `Response::send_to_client()` flushes the response to the client immediately and allows the WASM invocation to continue. This is not a small wiring change; it restructures how the application returns responses. - -```rust -async fn route_request(...) -> Result<(), Error> { - let geo_info = GeoInfo::from_request(&req); - - // Pre-routing — read only, no generation (matches GeoInfo pattern). - // EcContext stores client_ip internally (same req.get_client_ip_addr() - // already called by GeoInfo::from_request() above). - let ec_context_result = EcContext::read_from_request(&req, settings, geo_info.as_ref()); - let mut ec_context = match ec_context_result { - Ok(ctx) => ctx, - Err(e) => { - // Pre-routing failure — no route matched yet, but we still need to - // send an HTTP error response. Construct one and flush immediately. - log::error!("EcContext initialization failed: {e:?}"); - let mut response = to_error_response(&e); - response.send_to_client(); - return Ok(()); - } - }; - let kv = KvIdentityGraph::new(&settings.ec.ec_store); - let partner_store = PartnerStore::new(&settings.ec.partner_store); - let pull_sync_dispatcher = PullSyncDispatcher::new(settings.ec.pull_sync_concurrency); - - if let Some(mut response) = enforce_basic_auth(settings, &req) { - ec_finalize_response(settings, geo_info.as_ref(), &ec_context, &kv, &mut response); - response.send_to_client(); - return Ok(()); - } - - let path = req.get_path().to_string(); - let method = req.get_method().clone(); - - // Route dispatch — req is moved (consumed) inside the matching arm. - // is_organic tracks whether pull sync should fire (organic routes only — §10.2). - let mut is_organic = false; - let result = match (method, path.as_str()) { - // EC-specific routes — all read-only except /sync which takes &mut. - // /sync may assign fallback consent into ec_context.consent when the - // query param is the only signal — see §8.3. - (GET, "/sync") => handle_sync(settings, &kv, &partner_store, &req, &mut ec_context).await, - (GET, "/identify") => handle_identify(settings, &kv, &partner_store, &req, &ec_context).await, - (OPTIONS, "/identify") => cors_preflight_identify(settings, &req), - (POST, "/api/v1/sync") => handle_batch_sync(settings, &kv, &partner_store, req).await, - (POST, "/admin/partners/register") => handle_register_partner(settings, &partner_store, req).await, - - // /auction — EC-read-only; never generates EC. - // NOTE: handle_auction signature changes from (settings, orchestrator, req) to - // (settings, orchestrator, &kv, req, &ec_context) — this is a call-graph change, - // not just wiring. See §12 for the full auction integration. - (POST, "/auction") => handle_auction(settings, orchestrator, &kv, req, &ec_context).await, - - // Organic routes — generate EC if needed (best-effort, never 500s), then dispatch - (m, path) if integration_registry.has_route(&m, path) => { - is_organic = true; - ec_context.generate_if_needed(settings, &kv); - integration_registry.handle_proxy(&m, path, settings, req, &ec_context).await - }, - _ => { - is_organic = true; - ec_context.generate_if_needed(settings, &kv); - handle_publisher_request(settings, integration_registry, req, &ec_context) - }, - }; - - // Unwrap result — errors become error responses (matches existing pattern) - let mut response = result.unwrap_or_else(|e| to_error_response(&e)); - - // finalize_response runs on every route — enforces cookie write/deletion/last_seen - ec_finalize_response(settings, geo_info.as_ref(), &ec_context, &kv, &mut response); - - // Flush response to client; WASM continues for background pull sync. - response.send_to_client(); - - // Background pull sync — organic routes only (§10.2). Never fires on /sync, - // /identify, /auction, /api/v1/sync, or /admin/* routes. - // Fires outbound HTTP calls via send_async(), blocks on PendingRequest::wait(). - if is_organic { - if let (Some(ip), Ok(pull_partners)) = (ec_context.client_ip, partner_store.pull_enabled_partners()) { - pull_sync_dispatcher.dispatch_background(&ec_context, ip, &pull_partners, &kv); - } - } - - Ok(()) -} -``` - -The existing `finalize_response()` in `main.rs` becomes `ec_finalize_response()` with the extended signature that accepts `ec_context` and `kv`. The `#[fastly::main]` entrypoint changes to call `route_request()` and return `Ok(())` (the response is already sent via `send_to_client()`). - -`PullSyncDispatcher::dispatch_background` uses `Request::send_async()` to fire outbound HTTP calls, then calls `PendingRequest::wait()` (blocking) on each handle under `settings.ec.pull_sync_concurrency` concurrency. No async runtime is needed — this is synchronous blocking code running after `send_to_client()` has flushed the response. The Fastly WASM invocation stays alive until `dispatch_background` returns. This does not add latency to the user-facing response. - ---- - -## 18. Testing Strategy - -Follow the project's **Arrange-Act-Assert** pattern. Test both happy paths and error conditions. Use `expect()` with `"should ..."` messages. - -### 18.1 Unit tests - -Each module in `ec/` has a `#[cfg(test)]` module covering: - -| Module | Key test cases | -| --------------- | --------------------------------------------------------------------------------------------------------- | -| `identity.rs` | IPv4/IPv6 normalization, /64 truncation, HMAC determinism, output format | -| `finalize.rs` | `ec_finalize_response()`: cookie write on generation, deletion on withdrawal, `update_last_seen` debounce | -| `cookie.rs` | Cookie string format, Max-Age=0 for deletion, domain derivation | -| `kv.rs` | Serialization/deserialization roundtrip, CAS merge logic, metadata extraction | -| `partner.rs` | API key hash verification (constant-time), record serialization | -| `sync_pixel.rs` | All `ts_synced` redirect codes, 429 rate limit, return URL construction | -| `sync_batch.rs` | Status code selection (200/207/401/400/429), per-mapping rejection reasons, API-key rate limit | -| `pull_sync.rs` | Trigger conditions, null/404 no-op, dispatch limit | -| `identify.rs` | All response codes (200/403/204), degraded flag, `uids` filtering | - -### 18.2 Integration tests - -KV behavior is tested with Viceroy (local Fastly Compute simulator) using real KV store operations. Key scenarios: - -- Consent withdrawal: cookie deletion + tombstone write (`write_withdrawal_tombstone()`) + all EC response headers stripped — in same request -- Concurrent writes: CAS retry logic under simulated generation conflicts -- KV degraded: EC cookie still set when KV `create_or_revive()` fails (best-effort) -- Sync-then-identify flow: pixel sync writes partner ID, then `/identify` returns it - -**Eventually-consistent caveat:** Fastly KV does not guarantee read-after-write consistency. The sync→identify scenario may not be immediately visible on production — Viceroy may behave differently. Tests for this flow should use retry with backoff (up to 1s) and be documented as Viceroy-only consistency. Do not write assertions that assume immediate visibility after a KV write. - -### 18.3 JS tests (if applicable) - -If any JS changes are made for EC (e.g., publisher-side `/identify` fetch helper in `crates/js/`), use Vitest with `vi.hoisted()` for mocks. - ---- - -## 19. Implementation Order - -Suggested order to minimize risk and allow incremental testing. Each step should pass `cargo test --workspace` before the next begins. - -| Step | Scope | Deliverable | -| ---- | --------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | -| 1 | `ec/identity.rs` + constants + settings | `generate_ec()`, `normalize_ip()`, `EcContext` | -| 2 | `ec/finalize.rs` | `ec_finalize_response()` (cookie write, deletion, tombstone, last_seen) | -| 3 | `ec/cookie.rs` | Cookie creation, deletion, response header | -| 4 | `ec/kv.rs` | `KvIdentityGraph` CRUD with CAS | -| 5 | `ec/partner.rs` + `ec/admin.rs` | `PartnerStore`, `/admin/partners/register` | -| 6 | EC middleware in `main.rs`, `publisher.rs`, `registry.rs` | `EcContext::read_from_request()` pre-routing, `generate_if_needed()`, `ec_finalize_response()` | -| 7 | `ec/sync_pixel.rs` | `GET /sync` handler + route | -| 8 | `ec/identify.rs` | `GET /identify` handler + route | -| 9 | `ec/sync_batch.rs` | `POST /api/v1/sync` handler + route | -| 10 | `ec/pull_sync.rs` | Background pull sync dispatch (blocking, after `send_to_client()`) | -| 11 | Auction integration | EC injection into `user.id`, `user.eids`, `user.consent` | -| 12 | End-to-end integration tests | Viceroy-based flow tests | - ---- - -## 20. Epic and Stories - -### Epic: Implement Edge Cookie (EC) identity system - -Enable the trusted server to generate, persist, and serve a publisher-owned, -privacy-safe Edge Cookie (EC) that can be used for ID sync, identity lookup, -and auction decoration — without relying on third-party cookies. - -**Done when:** All 12 stories below are complete, `cargo test --workspace` and -`cargo clippy` pass with no warnings, and the end-to-end Viceroy flow tests -cover the full sync → identify → auction path. - -**Spec ref:** This document. PRD: `docs/internal/ssc-prd.md`. - ---- - -### Story 1 — EC generation and request context - -Implement the core EC data types, generation logic, and per-request context -struct that all subsequent stories depend on. - -**Scope:** `ec/identity.rs`, `ec/mod.rs`, `trusted-server.toml` `[ec]` section, -`Settings` struct update. - -**Acceptance criteria:** - -- `generate_ec(passphrase, ip)` produces a deterministic 71-char string: - 64-char lowercase hex hash + `.` + 6-char random alphanumeric suffix. - HMAC inputs are `normalize_ip(ip)` as message and `passphrase` as key. -- `normalize_ip()` truncates IPv6 to /64 (first 4 groups), passes IPv4 unchanged. -- IP is sourced from `req.get_client_ip_addr()` — no header fallback. -- `EcContext::read_from_request(req, settings, geo)` reads the `ts-ec` cookie - and `X-ts-ec` header. Sets `cookie_was_present`, `ec_was_present`, `ec_value`, - and `cookie_ec_value` (when header and cookie carry different valid EC values — - see §4.2 mismatch handling). Validates values via `ec_hash()` — malformed - values are treated as absent; if header is invalid, falls back to cookie. - Captures `client_ip` from `req.get_client_ip_addr()` (stored as - `Option` for pull sync use after `req` is consumed by routing). - Calls `build_consent_context()` with the EC hash as identity key and stores - the result as `consent: ConsentContext` (see §6.1.1). Does not generate. - Does not write to EC identity KV. (Note: `build_consent_context()` may write - to the consent KV store when an EC hash is available.) -- `EcContext::generate_if_needed(settings, kv)` generates a new EC when - `ec_value == None && allows_ec_creation(&consent)`, sets `ec_generated = true`, - and writes the initial KV entry via `kv.create_or_revive()` (best-effort). - Using `create_or_revive` (not `create`) ensures re-consent within the 24h - tombstone window recovers immediately. This function is best-effort: if - generation fails (e.g., missing client IP), it logs `warn` and returns - without setting `ec_generated`. It never returns an error — organic traffic - must not 500 on EC failure. -- `[ec]` settings block parses from TOML: `passphrase`, `ec_store`, - `partner_store`, `admin_token_hash`, `pull_sync_concurrency`. -- All unit tests in `identity.rs` pass (HMAC determinism, format, IP normalization). - -**Spec ref:** §2, §3, §4, §5.4, §14.1 - ---- - -### Story 2 — EC finalize response - -Implement `ec_finalize_response()` — the post-routing function that enforces -cookie writes, deletions, tombstones, and last-seen updates on every response. - -**Scope:** `ec/finalize.rs` (new file) - -**Acceptance criteria:** - -- `ec_finalize_response(settings, geo, ec_context, kv, response)` runs on every route. -- Consent gating uses the existing `allows_ec_creation()` — no new gating function. -- When `!allows_ec_creation(&consent) && cookie_was_present`: calls - `clear_ec_on_response()` (deletes cookie and strips all EC response headers) - and writes tombstone for each valid EC hash available. When the cookie is - malformed and no valid header exists, no tombstone is written — cookie - deletion alone enforces withdrawal (see §6.2). -- When `ec_was_present && !ec_generated && allows_ec_creation(&consent)`: calls - `kv.update_last_seen(ec_hash, now())` (debounced at 300s). If `cookie_ec_value` - is set (header/cookie mismatch), also calls `set_ec_on_response()` to reconcile - the browser cookie to the header-derived identity. -- When `ec_generated == true`: calls `set_ec_on_response()`. -- Unit tests cover all four branches: withdrawal (with and without valid hash), - returning-user last_seen + mismatch reconciliation, and new-EC generation. - -**Spec ref:** §5.4, §6.2 - ---- - -### Story 3 — EC cookie helpers - -Implement the low-level functions that create and delete the `ts-ec` cookie -and set EC response headers. These are called by `ec_finalize_response()` (Story 2). - -**Scope:** `ec/cookie.rs` - -**Acceptance criteria:** - -- `create_ec_cookie()` produces a cookie with `Domain=.{publisher.domain}`, - `Max-Age=31536000`, `SameSite=Lax; Secure`. `HttpOnly` is NOT set - (JS on the publisher page must be able to read the cookie). -- `delete_ec_cookie()` produces a cookie with `Max-Age=0`, same attributes. -- `set_ec_on_response()` sets `Set-Cookie` and `X-ts-ec` response headers. -- `clear_ec_on_response()` sets `Set-Cookie` with `Max-Age=0` **and** strips all - EC-related response headers: `X-ts-ec`, `X-ts-eids`, `X-ts-ec-consent`, - `x-ts-eids-truncated`, and any `X-ts-` headers. This prevents - leaking EC identity on consent-withdrawal responses where a handler may have - already set these headers before `ec_finalize_response()` runs. -- Unit tests cover cookie string format, Max-Age=0 deletion, domain derivation, - and header stripping (verify headers are removed after `clear_ec_on_response`). - -**Spec ref:** §5.1, §5.3, §5.4, §17 (ec_finalize_response) - ---- - -### Story 4 — KV identity graph - -Implement the KV read/write/delete layer for EC identity entries, including -CAS-based concurrent write protection and consent withdrawal delete. - -**Scope:** `ec/kv.rs` - -**Acceptance criteria:** - -- `KvIdentityGraph::get(ec_hash)` returns the deserialized entry and generation - marker as `Option<(KvEntry, u64)>`, or `None` if not found. -- `KvIdentityGraph::get_metadata(ec_hash)` returns `Option` for - cheap consent/country checks without streaming the full body. -- `KvIdentityGraph::create(ec_hash, &entry)` writes a new entry with - `consent.ok = true`. Returns `Err` if the key already exists (concurrent - create) or on KV error. No retry — callers handle conflicts. -- `KvIdentityGraph::create_or_revive(ec_hash, &entry)` creates a new entry OR - overwrites an existing tombstone (`consent.ok = false`) with a fresh entry; - no-ops if a live entry already exists. Called by `generate_if_needed()`. -- `KvIdentityGraph::update_last_seen(ec_hash, timestamp)` updates `last_seen` - without overwriting partner IDs (CAS merge), and only writes if the stored - value is more than 300s older than `timestamp` (debounce to avoid 1 write/sec - KV limit). Callers pass `now()` as `timestamp`. -- `KvIdentityGraph::write_withdrawal_tombstone(ec_hash)` sets `consent.ok = false`, - clears partner IDs, and applies a 24-hour TTL (see §6.2). Returns `Result` — - callers log `error` on failure and continue (cookie deletion is the primary - enforcement mechanism). -- `KvIdentityGraph::delete(ec_hash)` hard-deletes the entry — used only for IAB - data deletion requests, not for consent withdrawal (which uses tombstones). -- `kv.upsert_partner_id(ec_hash, partner_id, uid, timestamp)` writes to - `ids[partner_id]`, creating a minimal live root entry first if the key is - absent, and skips if existing `synced >= timestamp` (idempotent). -- KV schema matches §7 exactly (JSON roundtrip test). -- Unit tests cover CAS merge logic, tombstone write, tombstone error handling, - serialization/deserialization roundtrip, metadata extraction. - -**Spec ref:** §4, §5.4, §6.2 - ---- - -### Story 5 — Partner registry and admin endpoint - -Implement `PartnerRecord`, `PartnerStore`, and the admin registration endpoint -that operators use to onboard ID sync partners. - -**Scope:** `ec/partner.rs`, `ec/admin.rs`, router update - -**Acceptance criteria:** - -- `PartnerRecord` contains all fields from §13.1 including - `pull_sync_allowed_domains` and `batch_rate_limit`. -- `PartnerStore::get()`, `upsert()`, `find_by_api_key_hash()` operate on - `partner_store` KV. -- `pull_enabled_partners()` re-checks `pull_sync_enabled == true` on fetched - records so stale `_pull_enabled` index entries do not dispatch disabled partners. -- API key stored as SHA-256 hex; plaintext never written to KV. -- `verify_api_key()` uses constant-time comparison. -- `POST /admin/partners/register` validates `Authorization: Bearer ` inside - the handler against `settings.ec.admin_token_hash` (constant-time SHA-256 comparison). - Returns `401` if missing or invalid — before any request body is read. -- Admin endpoint validates: `pull_sync_url` hostname must be in - `pull_sync_allowed_domains` when set — returns `400` otherwise. -- Returns `201 Created` on new partner or `200 OK` on update, with an explicit - response DTO (see §13.2 step 6 — do NOT serialize full `PartnerRecord`). - Returns `400` on validation failure; `503` on KV failure. -- `/admin/partners/register` is **NOT** added to `Settings::ADMIN_ENDPOINTS` — - it uses bearer-token-in-handler auth, not `[[handlers]]` Basic Auth. -- The admin-route-scan test (`settings.rs:1504-1530`) must be updated to exclude - bearer-token-authed routes from its `ADMIN_ENDPOINTS` assertion. Add an exclusion - list (see §13.2 codebase invariant note). -- The `[[handlers]]` pattern in `trusted-server.toml` must be narrowed from - `"^/admin"` to `"^/admin/keys"` (see §13.2). -- Unit tests cover API key hash verification and record serialization. - -**Spec ref:** §13 - ---- - -### Story 6 — EC middleware integration - -Wire `EcContext` into the request pipeline following the two-phase model -(§5.4 and §17.1). `EcContext::read_from_request()` runs pre-routing like -`GeoInfo`; `generate_if_needed()` runs inside organic handlers only. - -**Scope:** `main.rs`, `publisher.rs`, `endpoints.rs`, `registry.rs` - -**Acceptance criteria:** - -- `EcContext::read_from_request()` is called before the route match on every - request, passed the existing `geo_info` (no duplicate geo header parsing). -- EC route handlers receive `ec_context` without EC generation. `/identify`, - `/auction`, `/api/v1/sync`, and `/admin/*` use read-only `&EcContext` and - never mutate it. **Exception:** `/sync` receives `&mut EcContext`; when the - consent query-param fallback applies (`ec_context.consent.is_empty()`), it - assigns the locally-decoded consent into `ec_context.consent` so that both - the sync write decision and `ec_finalize_response()` share the same effective - consent view. This prevents a same-request "write partner ID, then withdraw - EC" conflict. See §8.3 for full details. -- `/auction` consumes EC identity but never bootstraps it. -- `handle_publisher_request()` and `integration_registry.handle_proxy()` call - `ec_context.generate_if_needed(settings, &kv)` before their handler logic (best-effort, never 500s). -- `ec_finalize_response()` receives `ec_context` and `kv` and: - - Deletes the EC cookie and writes a withdrawal tombstone when `!allows_ec_creation(&consent) && cookie_was_present` (runs on all routes). - - Calls `kv.update_last_seen(ec_hash, now())` when `ec_was_present == true && ec_generated == false && allows_ec_creation(&consent)` (returning user with valid consent). - - Calls `set_ec_on_response()` when `ec_context.ec_generated == true`, and also - on returning-user mismatch reconciliation when `cookie_ec_value.is_some()`. -- `route_request()` return type changes from `Result` to - `Result<(), Error>`; response is flushed via `response.send_to_client()` instead - of being returned. The `#[fastly::main]` entrypoint must also change to match. - This is a structural refactor of the Fastly entrypoint, not an additive change — - see §17.1 for the full scope. -- `handle_auction()` signature changes to accept `&KvIdentityGraph` and `&EcContext` - (see §17.1 pseudocode comment). -- **Handler refactoring:** PR #479 removes `get_or_generate_synthetic_id()`, - `COOKIE_SYNTHETIC_ID`, and `X-Synthetic-*` headers from all handlers. This - epic completes the refactoring by replacing the internal `build_consent_context()` - calls with `ec_context.consent`: - - `handle_publisher_request()`, `handle_auction()`, and - `integration_registry.handle_proxy()` no longer call `build_consent_context()` - internally — they use `ec_context.consent` (built pre-routing). - - Identity comes from `ec_context.ec_value` (no synthetic fallback). -- `cargo test --workspace` passes with no regressions. - -**Spec ref:** §5, §17 - ---- - -### Story 7 — Pixel sync (`GET /sync`) - -Implement the pixel-based ID sync endpoint that partners use to write their -user ID against an EC hash. - -**Scope:** `ec/sync_pixel.rs`, router update - -**Acceptance criteria:** - -- Missing required query params (`partner`, `uid`, `return`) → `400`. -- No valid `ts-ec` cookie (missing or malformed) → redirect to - `{return}?ts_synced=0&ts_reason=no_ec`. -- Unknown `partner` ID → `400`. -- `return` URL hostname not in `partner.allowed_return_domains` → `400`. -- Consent uses `ec_context.consent`. The optional `consent` query param is a fallback - only: it is used exclusively when `ec_context.consent.is_empty()` returns `true` - — meaning no consent signals of any kind are present (no TCF string, no GPP - string, no US Privacy string, no AC string, no GPC, no decoded consent objects). - Use the `ConsentContext::is_empty()` method directly; do not reimplement the - check from this description. If consent KV fallback or any other pre-routing - source has already populated `ec_context.consent`, `is_empty()` is `false` and - the param is ignored. - When the fallback applies, decode the consent string locally into a - `ConsentContext` and **assign it into `ec_context.consent`** so that both - the sync write and `ec_finalize_response()` share the same effective consent - (prevents a same-request "write partner ID, then withdraw EC" conflict). - Do NOT re-call `build_consent_context()` (that would trigger consent KV writes). - Denied or absent → redirect to `{return}?ts_synced=0&ts_reason=no_consent`. -- Rate limit exceeded → `429 Too Many Requests` (no redirect). -- KV write failure → redirect to `{return}?ts_synced=0&ts_reason=write_failed`. -- `kv.upsert_partner_id()` creates a minimal live root entry first when the EC - exists in the cookie but the identity graph key is still missing because the - original best-effort `create_or_revive()` failed on generation. -- Success → redirect to `{return}?ts_synced=1`. -- Return URL construction correctly appends `&` or `?` based on existing query string. -- Rate counter key: `{partner_id}:{ec_hash}`, 1-hour window, via `fastly::erl::RateCounter`. -- Unit tests cover all redirect/response codes and return URL construction. - -**Spec ref:** §8 - ---- - -### Story 8 — Identity lookup (`GET /identify`) - -Implement the browser-facing endpoint that publishers call to retrieve the EC -hash and synced partner UIDs for the current user. - -**Scope:** `ec/identify.rs`, router update - -**Acceptance criteria:** - -- `!allows_ec_creation(consent)` (consent denied, regardless of EC presence) → `403 Forbidden`. - When EC is present but consent is denied, the handler returns `403` and - `ec_finalize_response()` deletes the cookie and writes a tombstone. -- No EC present (`ec_was_present == false`) and consent not denied → `204 No Content`. -- Valid EC, consent granted, KV read succeeds with entry → `200` with full JSON body - including `ec`, `consent`, `uids`, `eids`. -- Valid EC, consent granted, KV read succeeds but no entry (never synced or - `create_or_revive()` failed on generation) → `200` with `degraded: false`, - empty `uids`/`eids`. This is not an error — see §11.4. -- `uids` filtered to partners where `bidstream_enabled = true` and consent - granted. -- KV read error (store unavailable) → `200` with `degraded: true` and empty - `uids`/`eids`. -- No `Origin` header (server-side proxy): process normally, no CORS headers, no `403`. -- `Origin` header present and matches `publisher.domain` or subdomain: reflect in - `Access-Control-Allow-Origin` + `Vary: Origin`. -- `Origin` header present but does not match: `403`, no body. -- `OPTIONS /identify` preflight → `200` with CORS headers, no body. -- `generate_if_needed()` is never called — no new EC is generated. The handler - itself does not write cookies, but `ec_finalize_response()` may still delete - the cookie on withdrawal or reconcile it on header/cookie mismatch. -- Response time target: 30ms p95 (documented, not gate). -- Unit tests cover all response codes, degraded flag, `uids` filtering, - CORS origin validation. - -**Spec ref:** §11 - ---- - -### Story 9 — S2S batch sync (`POST /api/v1/sync`) - -Implement the server-to-server batch sync endpoint for partners to bulk-write -their UIDs against a list of EC hashes. - -**Scope:** `ec/sync_batch.rs`, router update - -**Acceptance criteria:** - -- Missing or invalid `Authorization: Bearer` → `401`. Auth uses index-based - lookup via `find_by_api_key_hash()` (§9.2) with constant-time hash verification. -- Auth KV lookup failure (store unavailable) → `503 Service Unavailable`. -- API-key rate limit exceeded (`batch_rate_limit` per partner per minute) → `429` - with `{ "error": "rate_limit_exceeded" }`. -- More than 1000 mappings → `400`. -- Per-mapping rejections: `invalid_ec_hash`, `ec_hash_not_found`, - `consent_withdrawn`, `kv_unavailable`. -- KV write failure aborts remaining mappings with `kv_unavailable`; partial - results returned as `207`. -- All mappings accepted → `200`. Any rejection → `207`. -- `kv.upsert_partner_id()` is idempotent: duplicate timestamp counted as - accepted, no error. -- Rate counter key: `batch:{partner_id}`, 1-minute window. -- Unit tests cover status code selection, all rejection reasons, and API-key - rate limit. - -**Spec ref:** §9 - ---- - -### Story 10 — Pull sync dispatch - -Implement the background pull sync dispatcher that calls partner resolution -endpoints after the response is flushed via `send_to_client()`. Uses -`send_async()` + `PendingRequest::wait()` (synchronous blocking, no async -runtime). Only fires on organic routes (§10.2). - -**Scope:** `ec/pull_sync.rs` - -**Acceptance criteria:** - -- Dispatch only when: EC present (including an EC generated on the current - organic request), consent granted, `pull_sync_enabled = true`, and either no - existing partner entry or existing `synced` is older than `pull_sync_ttl_sec`. -- Rate limit: `pull_sync_rate_limit` per EC hash per partner per hour; counter - key `pull:{partner_id}:{ec_hash}`. -- Maximum concurrent pulls per request: `settings.ec.pull_sync_concurrency` - (default 3). -- Before calling, validate `pull_sync_url` hostname is in - `pull_sync_allowed_domains`; skip and log `error` if not. -- Outbound request: `GET {pull_sync_url}?ec_hash={hash}&ip={ip}` with - `Authorization: Bearer {ts_pull_token}`. -- `{ "uid": null }` and `404` are no-ops — no KV write, no error logged above - `debug`. -- Any other non-200 → transient failure, no retry, no error above `warn`. -- Dispatch runs after `send_to_client()` — does not add latency to the - user-facing response. Uses `send_async()` + `PendingRequest::wait()` (blocking). -- Only fires on organic routes (`handle_publisher_request`, `handle_proxy`) — - never on `/sync`, `/identify`, `/auction`, `/api/v1/sync`, or `/admin/*`. -- Unit tests cover trigger conditions, null/404 no-op, domain allowlist check, - dispatch limit enforcement. - -**Spec ref:** §10 - ---- - -### Story 11 — Auction bidstream decoration - -Inject EC identity data into outbound OpenRTB bid requests for publishers with -`bidstream_enabled = true` partners. - -**Scope:** Auction handler (Mode B path in existing auction code) - -**Acceptance criteria:** - -- `user.id` set to `ec_context.ec_value` when EC present and consent granted. - No synthetic fallback — when no EC is present, `user.id` is omitted. -- `user.eids` populated with one entry per `bidstream_enabled` partner that - has a synced UID, using `partner.source_domain` and `partner.openrtb_atype`. -- `user.consent` set to `ec_context.consent.raw_tc_string` when present. -- No EID entry written for partners with no synced UID. -- KV read failure → `user.eids` omitted (empty); `user.id` still set from EC; - auction proceeds without EID data (no 5xx). -- No EC present → `user.id` omitted; `user.eids` is empty. Auction still runs. -- `X-Synthetic-*` response headers are not present (removed in PR #479). Only `X-ts-ec` is set. -- Unit tests cover EID structure, consent string threading, KV-degraded path, - and no-EC path (verify no synthetic fallback). - -**Spec ref:** §12 - ---- - -### Story 12 — End-to-end integration tests - -Write Viceroy-based integration tests covering the full identity lifecycle -across multiple handlers in a single simulated environment. - -**Scope:** `tests/` (integration test crate or new test module) - -**Acceptance criteria:** - -- **Full flow:** First-party page load → EC generated → pixel sync writes - partner UID → `/identify` returns that UID → auction includes EID. -- **Consent withdrawal:** Request with denied consent clears EC cookie and writes - a KV tombstone (`consent.ok = false`, 24h TTL) in the same request; subsequent - `/identify` with consent still denied returns `403` (consent denied → §11.4); - batch sync returns `consent_withdrawn` within the tombstone TTL. -- **KV create failure:** EC cookie is still set when `create_or_revive()` fails - (best-effort). Subsequent `/identify` returns `200` with `degraded: false` and - empty `uids`/`eids` (KV read succeeds — entry simply does not exist). -- **KV read failure:** `/identify` returns `200` with `degraded: true` and empty - `uids`/`eids` (store unavailable, entry might exist but can't be read). -- **Concurrent writes:** Two simultaneous EC creates for the same hash resolve - without data loss (CAS retry). -- **Rate limits:** Pixel sync returns `429` after `sync_rate_limit` is - exceeded; batch sync returns `429` after `batch_rate_limit` is exceeded. -- **Pull sync no-op:** Partner returning `{ "uid": null }` produces no KV - write and no error log. -- All tests pass under `cargo test --workspace` with Viceroy. - -**Spec ref:** §18.2 diff --git a/fastly.toml b/fastly.toml index 9d6c0f26..28961877 100644 --- a/fastly.toml +++ b/fastly.toml @@ -23,6 +23,14 @@ build = """ [local_server.backends] [local_server.kv_stores] + [[local_server.kv_stores.counter_store]] + key = "placeholder" + data = "placeholder" + + [[local_server.kv_stores.opid_store]] + key = "placeholder" + data = "placeholder" + [[local_server.kv_stores.creative_store]] key = "placeholder" data = "placeholder" @@ -30,6 +38,14 @@ build = """ [[local_server.kv_stores.consent_store]] key = "placeholder" data = "placeholder" + + [[local_server.kv_stores.ec_identity_store]] + key = "placeholder" + data = "placeholder" + + [[local_server.kv_stores.ec_partner_store]] + key = "placeholder" + data = "placeholder" [local_server.secret_stores] [[local_server.secret_stores.signing_keys]] key = "ts-2025-10-A" diff --git a/trusted-server.toml b/trusted-server.toml index 460c657e..7953c4f9 100644 --- a/trusted-server.toml +++ b/trusted-server.toml @@ -16,6 +16,8 @@ proxy_secret = "change-me-proxy-secret" [ec] passphrase = "trusted-server" +ec_store = "ec_identity_store" +partner_store = "ec_partner_store" # Custom headers to be included in every response # Allows publishers to include tags such as X-Robots-Tag: noindex