diff --git a/.gitignore b/.gitignore index 6efac56a..8aa23925 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ ui/ts/**/*.d.ts.map # Coverage reports generated by local or CI runs /coverage /solidity/coverage +/reth/.env /trading/artifacts /trading/node_modules /trading/ts/artifacts/contractArtifact.ts diff --git a/augurScan/README.md b/augurScan/README.md index 435739ed..ba452030 100644 --- a/augurScan/README.md +++ b/augurScan/README.md @@ -21,7 +21,7 @@ On Windows, run `start.bat` from this directory to start the same Compose comman Open . PostgreSQL is included and stored in the `augurscan-data` named volume. The website is available while historical backfill is running and reports the indexed block, its timestamp/age, observed head, lag, percentage complete, estimated time remaining, and network errors. Completion is measured from the effective start block stored for the index to the latest observed head. The ETA appears after the indexer or browser has observed enough forward progress to measure throughput. -The Compose services join the shared external `zoltar` network. When this repository's Erigon Compose project is running, set `SEPOLIA_RPC_URL=http://erigon:8545` to use it without exposing RPC beyond the host. +The Compose services join the shared external `zoltar` network. When this repository's Reth Compose project is running, set `SEPOLIA_RPC_URL=http://reth:8545` to use it without exposing RPC beyond the host. Set private or higher-capacity RPC endpoints in `.env` for reliable historical indexing. The included public defaults are convenient for evaluation but can rate-limit large backfills. On a fresh database, augurScan searches from `MAINNET_START_BLOCK` or `SEPOLIA_START_BLOCK` through the observed head for the earliest deployment whose history it tracks, then begins indexing at that deployment instead of walking earlier empty blocks. If none of those contracts is deployed yet, indexing waits at the block after the observed head. The conservative default floor of block `0` cannot omit protocol history; a later verified floor reduces historical bytecode reads. Before accepting a manifest change, augurScan verifies that every newly tracked deployment is within the stored index range. An earlier deployment stops with a rebuild error and leaves the current canonical index unchanged. An accepted address, label, or kind change resets that network to its stored effective start and durably replays the current manifest. diff --git a/erigon/Dockerfile b/erigon/Dockerfile deleted file mode 100644 index c8d30652..00000000 --- a/erigon/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM erigontech/erigon:v3.5.5 - -COPY --chmod=755 erigon/docker-entrypoint.sh /usr/local/bin/zoltar-erigon-entrypoint - -ENTRYPOINT ["/usr/local/bin/zoltar-erigon-entrypoint"] -CMD ["--help"] diff --git a/erigon/Dockerfile.dockerignore b/erigon/Dockerfile.dockerignore deleted file mode 100644 index 1f3e5880..00000000 --- a/erigon/Dockerfile.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -** -!erigon/docker-entrypoint.sh diff --git a/erigon/README.md b/erigon/README.md deleted file mode 100644 index 266e857e..00000000 --- a/erigon/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Sepolia archive node - -This container runs Erigon's embedded execution and consensus clients for Sepolia in archive mode. It retains all historical state and enables persisted receipts, so `eth_getLogs` can query every block from the start of the current month as well as older history. - -Erigon also writes its own process logs under `/home/erigon/.local/share/erigon/logs` in the persistent data volume. Those files survive container recreation. - -## Start - -From this directory, run: - -```bash -docker network inspect zoltar >/dev/null 2>&1 || docker network create zoltar -docker compose up --build -d -docker compose logs -f erigon -``` - -On Windows, run `start.bat` from this directory. It starts the same Compose project in the foreground and keeps the window open when Docker exits so that errors remain visible. - -The first startup lines print the RPC addresses. With the defaults, applications on the host connect to: - -```text -http://localhost:8545 -``` - -Other repository Compose services connect to `http://erigon:8545` through the shared external `zoltar` network. WebSockets use the same addresses with the `ws://` scheme. The host RPC binding is deliberately restricted to `127.0.0.1`; use a reverse proxy with authentication and TLS instead of exposing the debug and trace APIs directly to the internet. - -To use another host port, create an `.env` file beside `compose.yaml`: - -```dotenv -ERIGON_RPC_PORT=9545 -``` - -The printed host address would then be `http://localhost:9545`. The RPC remains available only from the Docker host and other Compose services. - -## Data and retention - -Chain data and Erigon's file logs live in the `erigon-sepolia-data` named volume. A normal `docker compose down` preserves it. Do not run `docker compose down --volumes` unless you intend to delete the archive and sync it again. - -An archive node retains all historical execution state rather than only a rolling window. This is intentionally broader than retaining logs from the first day of the current month and avoids a date-to-block boundary that would become stale. Initial synchronization downloads the historical archive and can take substantial time, disk space, and bandwidth; the RPC listener starts immediately, but historical requests are complete only after the relevant sync stages finish. - -The public P2P, Caplin discovery, and snapshot-download ports are published for reliable sync. The JSON-RPC port is only published on loopback. If a host firewall is enabled, allow inbound TCP/UDP `30303`, UDP `4000`, TCP `4001`, and TCP/UDP `42069` as appropriate for the host. - -Verify the node and its network after it has started: - -```bash -curl --fail-with-body \ - --header 'content-type: application/json' \ - --data '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}' \ - http://localhost:8545 -``` - -Sepolia's chain ID is returned as `0xaa36a7`. diff --git a/erigon/compose.yaml b/erigon/compose.yaml deleted file mode 100644 index f270285d..00000000 --- a/erigon/compose.yaml +++ /dev/null @@ -1,42 +0,0 @@ -services: - erigon: - build: - context: .. - dockerfile: erigon/Dockerfile - image: zoltar/sepolia-erigon:v3.5.5 - restart: unless-stopped - environment: - ERIGON_RPC_PUBLIC_PORT: ${ERIGON_RPC_PORT:-8545} - command: - - --chain=sepolia - - --datadir=/home/erigon/.local/share/erigon - - --prune.mode=archive - - --persist.receipts=true - - --http - - --http.addr=0.0.0.0 - - --http.port=8545 - - --http.api=eth,erigon,web3,net,debug,trace,txpool - - --http.vhosts=localhost,erigon - - --ws - - --log.dir.path=/home/erigon/.local/share/erigon/logs - - --log.dir.prefix=sepolia - - --mcp.disable - ports: - - "127.0.0.1:${ERIGON_RPC_PORT:-8545}:8545" - - "30303:30303/tcp" - - "30303:30303/udp" - - "4000:4000/udp" - - "4001:4001/tcp" - - "42069:42069/tcp" - - "42069:42069/udp" - volumes: - - erigon-sepolia-data:/home/erigon/.local/share/erigon - stop_grace_period: 2m - -volumes: - erigon-sepolia-data: - -networks: - default: - name: zoltar - external: true diff --git a/erigon/docker-entrypoint.sh b/erigon/docker-entrypoint.sh deleted file mode 100644 index 58e19768..00000000 --- a/erigon/docker-entrypoint.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -set -eu - -if [ "$#" -eq 0 ] || [ "${1#-}" != "$1" ]; then - set -- erigon "$@" -fi - -rpc_public_port=${ERIGON_RPC_PUBLIC_PORT:-8545} -month_start=$(date -u '+%Y-%m-01T00:00:00Z') - -printf '%s\n' \ - "Sepolia archive retention: all chain history (including logs since ${month_start})" \ - "RPC endpoint from this host: http://localhost:${rpc_public_port}" \ - 'RPC endpoint from another Compose service: http://erigon:8545' - -exec "$@" diff --git a/reth/.env.example b/reth/.env.example new file mode 100644 index 00000000..797dac5b --- /dev/null +++ b/reth/.env.example @@ -0,0 +1,7 @@ +# Decimal Sepolia block number of the first block whose receipts must be kept. +# Resolve this once for the required UTC cutoff; do not advance it on restarts. +RETH_RECEIPTS_START_BLOCK= + +# Host-only JSON-RPC ports. Compose services use reth:8545 and reth:8546. +RETH_RPC_PORT=8545 +RETH_WS_PORT=8546 diff --git a/reth/README.md b/reth/README.md new file mode 100644 index 00000000..f653ab74 --- /dev/null +++ b/reth/README.md @@ -0,0 +1,96 @@ +# Sepolia Reth node + +This Compose project runs Reth as the Sepolia execution client and Lighthouse as its consensus client. Reth prunes execution history before a fixed boundary while preserving every block body, receipt, and transaction lookup from that block onward. Consequently, `eth_getLogs` remains available from that block even as the node continues running. + +## Choose the retention boundary + +Copy the environment template before the first start: + +```bash +cp .env.example .env +``` + +Resolve the first Sepolia block whose timestamp is at or after the required UTC boundary—for example, `2026-08-01T00:00:00Z`—using a trusted Sepolia RPC or block explorer. With Foundry installed, the lookup for that example is: + +```bash +cast find-block --rpc-url https://your-trusted-sepolia-rpc.example 1785542400 +``` + +Confirm whether the closest block is immediately before or after the cutoff, adjust to the first block at or after it, and put that decimal block number in `.env`: + +```dotenv +RETH_RECEIPTS_START_BLOCK=12345678 +``` + +The number above is an example, not the August 2026 Sepolia boundary. Verify the selected block and its parent before starting: + +```bash +curl --fail-with-body \ + --header 'content-type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_getBlockByNumber","params":["0xbc614e",false]}' \ + https://your-trusted-sepolia-rpc.example +``` + +The selected block's timestamp must be at or after the cutoff, and the preceding block's timestamp must be before it. + +> [!WARNING] +> Keep `RETH_RECEIPTS_START_BLOCK` fixed on every restart. Raising it authorizes Reth to permanently delete additional block bodies, receipts, and transaction lookups. + +## Start + +From this directory, run: + +```bash +docker network inspect zoltar >/dev/null 2>&1 || docker network create zoltar +docker compose config +docker compose up -d +docker compose logs -f reth lighthouse +``` + +On Windows, copy `.env.example` to `.env`, set the retention block, and run `start.bat`. + +Applications on the host use `http://localhost:8545` or `ws://localhost:8546`. Other repository Compose services on the external `zoltar` network use `http://reth:8545` or `ws://reth:8546`. Both host ports bind only to loopback; use an authenticated TLS reverse proxy if remote access is required. + +Set `RETH_RPC_PORT` or `RETH_WS_PORT` in `.env` to change the host ports without changing the container endpoints. + +Allow inbound TCP/UDP `30303` and UDP `9200` for Reth peer discovery, plus TCP/UDP `9000` and UDP `9001` for Lighthouse discovery and QUIC. The JSON-RPC ports do not need public firewall rules because they bind only to host loopback. + +## Data and pruning + +The `reth-sepolia-data` volume contains execution data and bounded Reth process logs. `lighthouse-sepolia-data` contains consensus data. The `engine-jwt` volume contains the randomly generated Engine API secret shared by both clients. A normal `docker compose down` preserves all three volumes. + +The pruning configuration: + +- retains block bodies, receipts, and transaction-hash lookups from `RETH_RECEIPTS_START_BLOCK` forward; +- discards sender-recovery data after it is no longer required; +- retains a rolling 10,064-block window of account and storage history; +- retains current state, so current RPC reads and transaction submission continue working. + +This supports historical `eth_getLogs` and receipt lookup from the configured boundary. It does not support old state queries or traces outside the rolling state-history window. Block, transaction, and receipt storage will continue growing because the requested event history is retained permanently. + +> [!WARNING] +> Never run `docker compose down --volumes` unless all Reth and Lighthouse data may be deleted. Never move the retention boundary forward unless the additional block, transaction, and receipt history may be irreversibly deleted. + +## Migration from Erigon + +The old Erigon database cannot be reused by Reth. Bring this project up with its new volumes and leave the old Erigon Compose project and volume stopped but intact until Reth is fully synchronized and verified. + +During migration, only one execution client can publish host port `8545` and P2P port `30303`. Either stop Erigon before starting this project or temporarily assign different host/P2P ports. Because the checked-in deployment uses fixed P2P ports, the simplest safe sequence is: + +1. Stop Erigon with `docker compose down` from the old checkout. Do not pass `--volumes`. +2. Start this Reth project. +3. Wait for both clients to synchronize. +4. Verify the chain ID, sync status, and logs spanning the retention boundary. +5. Point dependent services from `http://erigon:8545` to `http://reth:8545`. +6. Delete the old Erigon volume only after an explicit backup/rollback decision. + +Verify the execution endpoint: + +```bash +curl --fail-with-body \ + --header 'content-type: application/json' \ + --data '{"jsonrpc":"2.0","id":1,"method":"eth_chainId","params":[]}' \ + http://localhost:8545 +``` + +Sepolia returns `0xaa36a7`. Then query `eth_getLogs` once from the configured start block through a small known range and again across recent blocks. Do not retire Erigon until both return the expected events and `eth_syncing` reports that Reth has reached the head. diff --git a/reth/compose.yaml b/reth/compose.yaml new file mode 100644 index 00000000..9eb50616 --- /dev/null +++ b/reth/compose.yaml @@ -0,0 +1,91 @@ +services: + jwt-init: + image: busybox:1.37.0 + command: + - sh + - -ec + - | + if [ ! -s /jwt/jwt.hex ]; then + umask 077 + head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' > /jwt/jwt.hex + fi + volumes: + - engine-jwt:/jwt + restart: "no" + + reth: + image: ghcr.io/paradigmxyz/reth:v2.2.0 + restart: unless-stopped + depends_on: + jwt-init: + condition: service_completed_successfully + command: + - node + - --chain=sepolia + - --datadir=/data + - --prune.sender-recovery.full + - --prune.transaction-lookup.before=${RETH_RECEIPTS_START_BLOCK:?Set RETH_RECEIPTS_START_BLOCK in reth/.env} + - --prune.receipts.before=${RETH_RECEIPTS_START_BLOCK:?Set RETH_RECEIPTS_START_BLOCK in reth/.env} + - --prune.account-history.distance=10064 + - --prune.storage-history.distance=10064 + - --prune.bodies.before=${RETH_RECEIPTS_START_BLOCK:?Set RETH_RECEIPTS_START_BLOCK in reth/.env} + - --http + - --http.addr=0.0.0.0 + - --http.port=8545 + - --http.api=eth,web3,net,debug,trace,txpool + - --ws + - --ws.addr=0.0.0.0 + - --ws.port=8546 + - --ws.api=eth,web3,net,debug,trace,txpool + - --authrpc.addr=0.0.0.0 + - --authrpc.port=8551 + - --authrpc.jwtsecret=/jwt/jwt.hex + - --log.file.directory=/data/logs + - --log.file.name=sepolia.log + ports: + - "127.0.0.1:${RETH_RPC_PORT:-8545}:8545" + - "127.0.0.1:${RETH_WS_PORT:-8546}:8546" + - "30303:30303/tcp" + - "30303:30303/udp" + - "9200:9200/udp" + volumes: + - reth-sepolia-data:/data + - engine-jwt:/jwt:ro + stop_grace_period: 2m + + lighthouse: + image: sigp/lighthouse:v8.2.1 + restart: unless-stopped + depends_on: + jwt-init: + condition: service_completed_successfully + reth: + condition: service_started + command: + - lighthouse + - bn + - --network=sepolia + - --datadir=/data + - --execution-endpoint=http://reth:8551 + - --execution-jwt=/jwt/jwt.hex + - --checkpoint-sync-url=https://sepolia.checkpoint-sync.ethpandaops.io + - --http + - --http-address=0.0.0.0 + ports: + - "9000:9000/tcp" + - "9000:9000/udp" + - "9001:9001/udp" + volumes: + - lighthouse-sepolia-data:/data + - engine-jwt:/jwt:ro + stop_grace_period: 2m + +volumes: + engine-jwt: + lighthouse-sepolia-data: + reth-sepolia-data: + +networks: + default: + name: zoltar + external: true diff --git a/erigon/start.bat b/reth/start.bat similarity index 54% rename from erigon/start.bat rename to reth/start.bat index ad1c53b0..40c89f71 100644 --- a/erigon/start.bat +++ b/reth/start.bat @@ -1,7 +1,13 @@ @echo off pushd "%~dp0" || exit /b 1 +if not exist .env ( + echo Copy .env.example to .env and set RETH_RECEIPTS_START_BLOCK first. + popd + pause + exit /b 1 +) docker network inspect zoltar >nul 2>&1 || docker network create zoltar || exit /b 1 -docker compose up --build --force-recreate +docker compose up --force-recreate set "exit_code=%errorlevel%" popd pause