From 77348e7c69427529a42306836bbacc4d8cd0ac18 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Mon, 1 Jun 2026 13:46:41 -0700 Subject: [PATCH 1/4] feat(cells): Wire devservices ingest path for cell-routing mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cell-routing` mode in devservices now uses a two-relay chain that exercises Relay's advertised-upstream routing locally — entirely from sentry's devservices, leaving the relay repo and default `relay` config untouched: - relay-cell (:7900): processing terminus; registers with Sentry and advertises itself as the upstream for downstream relays. - relay-edge (:7901): connects to relay-cell via the synapse-ingest-router:3000. Project envelops are then forwarded directly to the advertised upstream it reads from relay-cell's project config, bypassing synapse. - also runs taskbroker + taskworker (build project configs) and ingest-events + post-process-forwarder-errors so events ingest end-to-end into a visible issue. - bin/send-cell-test-event.py: sends a test event through the edge. The relay credentials for the new `relay-edge` and `relay-cell` services were generated with the `relay credentials generate --stdout` method. The end to end path also depends on this change to Synapse's dev config https://github.com/getsentry/synapse/pull/134 --- bin/send-cell-test-event.py | 57 +++++++++++++++ devservices/config.yml | 71 ++++++++++++++++++- .../config/relay-cell-credentials.json | 5 ++ devservices/config/relay-cell.yml | 35 +++++++++ .../config/relay-edge-credentials.json | 5 ++ devservices/config/relay-edge.yml | 24 +++++++ 6 files changed, 196 insertions(+), 1 deletion(-) create mode 100755 bin/send-cell-test-event.py create mode 100644 devservices/config/relay-cell-credentials.json create mode 100644 devservices/config/relay-cell.yml create mode 100644 devservices/config/relay-edge-credentials.json create mode 100644 devservices/config/relay-edge.yml diff --git a/bin/send-cell-test-event.py b/bin/send-cell-test-event.py new file mode 100755 index 000000000000..0cbf9ee35432 --- /dev/null +++ b/bin/send-cell-test-event.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +"""Send a test event through the cell-routing edge relay using the Sentry SDK. + +Pair with `devservices up --mode cell-routing`. Resolves a project key from the +local dev database (the internal project, id 1, which every dev install +bootstraps — override with PROJECT_ID), points a DSN at the edge relay on :7901, +and captures an event with sentry_sdk. +The edge reads the advertised upstream from its project config and forwards the +envelope there (relay-cell), which processes it to Kafka -> Sentry. + +Unlike the curl helper, this exercises the real SDK envelope path +(/api//envelope/), which is closer to how an actual client talks to Relay. + +Usage (with the dev env active): + bin/send-cell-test-event.py + PROJECT_ID=42 bin/send-cell-test-event.py + TARGET=127.0.0.1:7900 bin/send-cell-test-event.py # override: e.g. straight to relay-cell +""" + +from __future__ import annotations + +import os + +from sentry.runner import configure + +configure() + +from sentry.models.projectkey import ProjectKey # noqa: E402 + +# Defaults to the edge relay's address from devservices/config.yml (cell-routing +# mode). Override TARGET to send elsewhere, e.g. straight to relay-cell on :7900. +target = os.environ.get("TARGET", "127.0.0.1:7901") +# The internal project (id 1) is bootstrapped for every dev install. +project_id = os.environ.get("PROJECT_ID", "1") + +key = ProjectKey.objects.filter(project_id=project_id).first() +if key is None: + raise SystemExit( + f"no project key found for project {project_id} in dev db " + "(is the devserver bootstrapped? try PROJECT_ID=)" + ) + +# Point the DSN host at the edge relay rather than at Sentry directly. +dsn = f"http://{key.public_key}@{target}/{key.project_id}" +print(f"Using DSN {dsn}") +print("(watch the edge route it: docker logs -f sentry-relay-edge-1)") + +import sentry_sdk # noqa: E402 + +# Send via an isolated scope + client so we don't replace Sentry's own global SDK. +client = sentry_sdk.Client(dsn=dsn, default_integrations=False, traces_sample_rate=0) +with sentry_sdk.new_scope() as scope: + scope.set_client(client) + event_id = scope.capture_message("cell-routing test", level="error") +client.flush(timeout=5) + +print(f"sent event {event_id} via {target}") diff --git a/devservices/config.yml b/devservices/config.yml index 176478efee9d..e6d211ff6783 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -100,6 +100,13 @@ x-sentry-service-config: description: Memcached used for caching spotlight: description: Spotlight server for local debugging + # Local Relays used only by the `cell-routing` mode to exercise Relay's + # advertised-upstream routing without touching the relay repo or the default + # `relay` config. relay-cell replaces the plain `relay` in this mode. + relay-cell: + description: Processing Relay terminus that advertises an upstream (cell-routing only) + relay-edge: + description: Downstream Relay that routes on the advertised upstream (cell-routing only) objectstore: description: Storage for files and blobs remote: @@ -294,7 +301,22 @@ x-sentry-service-config: post-process-forwarder-errors, ] minimal: [postgres, snuba] - cell-routing: [postgres, snuba, relay, spotlight, objectstore, synapse] + cell-routing: + [ + postgres, + snuba, + redis, + taskbroker, + spotlight, + objectstore, + synapse, + taskworker, + taskworker-scheduler, + ingest-events, + post-process-forwarder-errors, + relay-cell, + relay-edge, + ] # The minimal set of services Symbolicator needs to run # Sentry integration tests. symbolicator-tests: [postgres, snuba, objectstore] @@ -491,6 +513,53 @@ services: - devservices labels: - orchestrator=devservices + # Relay chain for the `cell-routing` mode. These exercise Relay's + # advertised-upstream routing entirely from sentry's devservices: the + # `advertised_upstream` value lives in relay-cell.yml (mounted only here), so + # the default `relay` service and its config are untouched. relay-cell is the + # processing terminus (replaces plain `relay` here); relay-edge forwards. + relay-cell: + image: ghcr.io/getsentry/relay:nightly + command: [run, --config, /etc/relay] + healthcheck: + test: + ['CMD', '/bin/relay', '--config', '/etc/relay', 'healthcheck', '--mode', 'live'] + interval: 5s + timeout: 5s + retries: 3 + ports: + - 127.0.0.1:7900:7900 + volumes: + - ./config/relay-cell.yml:/etc/relay/config.yml + - ./config/relay-cell-credentials.json:/etc/relay/credentials.json + extra_hosts: + - host.docker.internal:host-gateway + networks: + - devservices + labels: + - orchestrator=devservices + restart: unless-stopped + relay-edge: + image: ghcr.io/getsentry/relay:nightly + command: [run, --config, /etc/relay] + healthcheck: + test: + ['CMD', '/bin/relay', '--config', '/etc/relay', 'healthcheck', '--mode', 'live'] + interval: 5s + timeout: 5s + retries: 3 + ports: + - 127.0.0.1:7901:7901 + volumes: + - ./config/relay-edge.yml:/etc/relay/config.yml + - ./config/relay-edge-credentials.json:/etc/relay/credentials.json + extra_hosts: + - host.docker.internal:host-gateway + networks: + - devservices + labels: + - orchestrator=devservices + restart: unless-stopped # Taskbroker in passthrough mode for ingest-profiles topic (STREAM-1041). # This is a local service rather than a remote dependency because devservices # doesn't support passing custom environment variables to remote services (DI-1956). diff --git a/devservices/config/relay-cell-credentials.json b/devservices/config/relay-cell-credentials.json new file mode 100644 index 000000000000..dd442340fb13 --- /dev/null +++ b/devservices/config/relay-cell-credentials.json @@ -0,0 +1,5 @@ +{ + "secret_key": "YUxl8BCg4kay_7qpDGm6l2p0roa7Pxuk2cRm1qeQrTY", + "public_key": "8lXBuHpE-dFoTblfKkp9BqjIZl33dyaGhmyjuZLB6aY", + "id": "01ff20ff-95f8-49db-b3f8-71f0caaa054d" +} diff --git a/devservices/config/relay-cell.yml b/devservices/config/relay-cell.yml new file mode 100644 index 000000000000..f96e07beee47 --- /dev/null +++ b/devservices/config/relay-cell.yml @@ -0,0 +1,35 @@ +--- +# Processing Relay terminus for the `cell-routing` dev mode. This is the +# cell-routing replacement for the default `relay` service: same role (the +# processing relay next to Sentry, writing to Kafka) plus an advertised upstream. +# +# Chain: event -> relay-edge -> relay-cell -> Kafka -> Sentry +# +# `advertised_upstream` is the upstream relay-cell tells the downstream relay-edge +# to forward each project's envelopes/metrics to. In single-cell dev the only +# cell is relay-cell itself, so it points back here via the host-published port +# (a distinct address from relay-edge's default upstream), which keeps ingestion +# working end-to-end while making the advertised-upstream override observable in +# relay-edge's logs. +# +# This file is only mounted by the `cell-routing` mode, so `advertised_upstream` +# never affects default dev. +relay: + upstream: 'http://host.docker.internal:8001/' + advertised_upstream: 'http://host.docker.internal:7900/' + host: 0.0.0.0 + port: 7900 +logging: + # Verbose so you can see the forwarded envelope arrive and get produced to Kafka. + # Drop back to INFO once routing is validated. + level: trace + enable_backtraces: false +limits: + shutdown_timeout: 0 +processing: + enabled: true + kafka_config: + - {name: 'bootstrap.servers', value: 'kafka:9093'} + # Align with Relay's default `max_envelope_size` (200 MiB -> 209,715,200 bytes). + - {name: 'message.max.bytes', value: 209715200} + redis: redis://redis:6379 diff --git a/devservices/config/relay-edge-credentials.json b/devservices/config/relay-edge-credentials.json new file mode 100644 index 000000000000..42907b1b218c --- /dev/null +++ b/devservices/config/relay-edge-credentials.json @@ -0,0 +1,5 @@ +{ + "secret_key": "IaxE7cSgxBhE9Gtr11Y8iaWaXLBT4-_1r7z-j_stxE0", + "public_key": "NxO7wFbVRLhTgV6T4pSpQoOdplYLipExVQyNhdqM0lE", + "id": "7fb4f675-d94c-4e3e-b76b-cb724e7670f3" +} diff --git a/devservices/config/relay-edge.yml b/devservices/config/relay-edge.yml new file mode 100644 index 000000000000..366deca57023 --- /dev/null +++ b/devservices/config/relay-edge.yml @@ -0,0 +1,24 @@ +--- +# Downstream "edge" Relay for the `cell-routing` dev mode. It forwards traffic +# rather than processing it (no Kafka/Redis needed). Send test events here. +# +# Register + project config: relay-edge -> synapse-ingest-router -> relay-cell (+ Sentry) +# Envelopes (once advertised): relay-edge -> relay-cell directly (bypasses synapse) +# +# relay-edge registers and fetches project configs through the synapse +# ingest-router (its `upstream`). The ingest-router passes relay-cell's config +# through unchanged, including the top-level `upstream` field relay-cell injects +# from its `advertised_upstream`. relay-edge then sends envelopes/metrics +# directly to that advertised upstream (relay-cell), bypassing synapse and +# reusing its existing relay credentials. +relay: + upstream: 'http://synapse-ingest-router:3000/' + host: 0.0.0.0 + port: 7901 +logging: + # Verbose so the upstream forwarding (to the advertised upstream) is visible. + # Drop back to INFO once routing is validated. + level: trace + enable_backtraces: false +limits: + shutdown_timeout: 0 From 33cadc25e532bfac7bc4cf53beaecd92cab976f6 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Tue, 2 Jun 2026 12:04:52 -0700 Subject: [PATCH 2/4] allow print --- bin/send-cell-test-event.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/send-cell-test-event.py b/bin/send-cell-test-event.py index 0cbf9ee35432..8d09f987c10b 100755 --- a/bin/send-cell-test-event.py +++ b/bin/send-cell-test-event.py @@ -17,6 +17,9 @@ TARGET=127.0.0.1:7900 bin/send-cell-test-event.py # override: e.g. straight to relay-cell """ +# CLI helper: printing to the terminal is the whole point. +# ruff: noqa: T201 + from __future__ import annotations import os From 9841df41d9deb863772fd8163cf56cfeb9c5f6d6 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Tue, 2 Jun 2026 12:13:13 -0700 Subject: [PATCH 3/4] ensure only active key selected --- bin/send-cell-test-event.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/send-cell-test-event.py b/bin/send-cell-test-event.py index 8d09f987c10b..9d81f6539322 100755 --- a/bin/send-cell-test-event.py +++ b/bin/send-cell-test-event.py @@ -28,6 +28,7 @@ configure() +from sentry.models.project import Project # noqa: E402 from sentry.models.projectkey import ProjectKey # noqa: E402 # Defaults to the edge relay's address from devservices/config.yml (cell-routing @@ -35,11 +36,11 @@ target = os.environ.get("TARGET", "127.0.0.1:7901") # The internal project (id 1) is bootstrapped for every dev install. project_id = os.environ.get("PROJECT_ID", "1") - -key = ProjectKey.objects.filter(project_id=project_id).first() +project = Project.objects.filter(id=project_id).first() +key = ProjectKey.get_default(project) if project else None if key is None: raise SystemExit( - f"no project key found for project {project_id} in dev db " + f"no active store key found for project {project_id} in dev db " "(is the devserver bootstrapped? try PROJECT_ID=)" ) From db33b7ccb4e36156e944c149e76aed2aa2be0cd1 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Tue, 2 Jun 2026 12:19:05 -0700 Subject: [PATCH 4/4] flake8 --- bin/send-cell-test-event.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/send-cell-test-event.py b/bin/send-cell-test-event.py index 9d81f6539322..844a8e8b80dc 100755 --- a/bin/send-cell-test-event.py +++ b/bin/send-cell-test-event.py @@ -17,8 +17,10 @@ TARGET=127.0.0.1:7900 bin/send-cell-test-event.py # override: e.g. straight to relay-cell """ -# CLI helper: printing to the terminal is the whole point. +# CLI helper: printing to the terminal is the whole point. `print` is flagged by +# both linters under different codes, each needing its own file-level directive. # ruff: noqa: T201 +# flake8: noqa: S002 from __future__ import annotations