diff --git a/.dockerignore b/.dockerignore index 9550d54..e705b77 100644 --- a/.dockerignore +++ b/.dockerignore @@ -24,7 +24,8 @@ deploy/standalone/output/ # dedicated Docker stages. src/distill-fs/target/ src/sandboxd/output/ -src/yuanrong/ +src/yuanrong/* +!src/yuanrong/LICENSE # Python caches and package outputs. **/__pycache__/ diff --git a/AGENTS.md b/AGENTS.md index 0dbedbd..52f0a66 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -294,6 +294,12 @@ the sandbox bridge. YuanRong receives `INSTANCE_IP` in Kubernetes or the default-route interface address in standalone mode; `AKERNEL_NODE_IP` is the explicit override for multi-homed environments. +Node deployments keep openYuanRong's in-node address aligned with sandboxd's +actual bridge state. Kubernetes renders `node.sandboxIPRange` into sandboxd's +`ip_range`; standalone reads the same setting from `sandboxd_config.toml` when +preparing networking. The YuanRong bootstrap waits with a bounded timeout for +`sandbox0` and passes its assigned IPv4 address to `yr start --local_ip`. + The standalone sandboxd filestore is a loop-mounted XFS image under the bind-mounted `deploy/standalone/data/` directory. Explicit `storage_mb` quotas use this local-disk filestore; omitting `storage_mb` retains the configured diff --git a/builder/config/yr/config.toml.jinja b/builder/config/yr/config.toml.jinja new file mode 100644 index 0000000..aa9e955 --- /dev/null +++ b/builder/config/yr/config.toml.jinja @@ -0,0 +1,181 @@ +{# + AKernel runtime profile template. + + Deployment values are resolved from the CLI-provided env mapping. Strings + containing Python CLI runtime expressions remain escaped for its second + rendering pass. +#} +{% macro toml_string(value) -%} +"{%- for char in value -%} +{%- set encoded = char | tojson -%} +{{- char if encoded | length == 14 else encoded[1:-1] -}} +{%- endfor -%}" +{%- endmacro %} +{% set role = env["AKERNEL_ROLE"] %} +{% set cluster_roles = ["master", "frontend"] %} +{% set node_roles = ["node", "standalone"] %} +{% if role not in cluster_roles + node_roles %} +{{ unsupported_akernel_role }} +{% endif %} +{% set is_cluster = role in cluster_roles %} +{% set is_node = role in node_roles %} +{% set is_agent = role == "node" %} +{% set is_standalone = role == "standalone" %} +{% set has_master_mode = not is_agent %} +{% set enable_metrics = env.get("ENABLE_METRICS", "false") | lower %} +{% set enable_trace = env.get("ENABLE_TRACE", "false") | lower %} +{% set traefik_enable_tls = "true" if env.get("TRAEFIK_ENABLE_TLS", "false") | lower == "true" else "false" %} +{% set configured_host_ip = env.get("YR_NODE_IP", "") | trim %} +{% set configured_local_ip = env.get("YR_LOCAL_IP", "") | trim %} +{% set enable_faas_frontend = env.get("ENABLE_FAAS_FRONTEND", "true") | lower %} +{% set enable_iam_server = env.get("ENABLE_IAM_SERVER", "true") | lower %} +{% set runtime_hostname = "{{ hostname }}" %} + +# Generated at startup from builder/config/yr/config.toml.jinja. + +[values] +node_id = "{{ runtime_hostname }}" +{% if configured_host_ip %} +host_ip = {{ toml_string(configured_host_ip) }} +{% endif %} +{% if configured_local_ip %} +local_ip = {{ toml_string(configured_local_ip) }} +{% endif %} +{% if is_cluster %} +cpu_num = 1 +memory_num = 3904 +shared_memory_num = 4096 +deploy_path = "/home/yuanrong/master" +{% endif %} + +[values.fs.log] +path = "{{ '/home/yuanrong/master/log' if is_cluster else '/home/yuanrong/logs' }}" + +[values.fs.tls] +base_path = "/home/yuanrong/.cert" + +{% if not is_standalone %} +[values.etcd] +enable_multi_master = true +{% endif %} + +{% if is_cluster or is_agent %} +[[values.etcd.address]] +ip = {{ toml_string(env["ETCD_ADDRESS"]) }} +port = {{ env.get("ETCD_PORT", "2379") }} +peer_port = {{ env.get("ETCD_PEER_PORT", "2378") }} +{% endif %} + +[values.frontend] +ssl_enable = true +client_auth_type = "NoClientCert" +frontend_lease_bypass = true +enable_function_token_auth = true +enable_func_token_auth = true +{% if role == "frontend" %} +meta_service_address = {{ toml_string(env["META_SERVICE_ADDRESS"]) }} +{% endif %} +iam_server_address = "127.0.0.1:31113" + +[values.meta_service] +port = 31111 + +{% if has_master_mode %} +[mode.master] +etcd = {{ "true" if is_standalone else "false" }} +ds_master = {{ "true" if is_standalone else "false" }} +frontend = {{ enable_faas_frontend if role == "master" else "true" if role in ["frontend", "standalone"] else "false" }} +function_master = {{ "true" if role in ["master", "standalone"] else "false" }} +function_scheduler = false +meta_service = {{ "true" if role in ["master", "standalone"] else "false" }} +iam_server = {{ enable_iam_server if role == "master" else "true" if role in ["frontend", "standalone"] else "false" }} +{% endif %} + +[ds_worker.args] +{% if is_cluster %} +rpc_thread_num = 128 +{% endif %} +{% if is_node %} +node_timeout_s = 30 +client_dead_timeout_s = 60 +heartbeat_interval_ms = 1000 +node_dead_timeout_s = 120 +log_dir = "/home/yuanrong/logs" +{% endif %} + +[function_master.args] +services_path = "/home/yuanrong/deploy/process/services.yaml" +metrics_config_file = "/home/yuanrong/metrics/metrics_config.json" +traefik_enable_tls = {{ traefik_enable_tls }} +traefik_http_entry_point = {{ toml_string(env.get("TRAEFIK_HTTP_ENTRYPOINT", "websecure")) }} +traefik_forward_timeout_ms = 3000 +{% if enable_metrics == "true" and (is_cluster or is_standalone) %} +enable_metrics = true +{% endif %} +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +enable_trace = true +trace_config = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} +{% if is_cluster %} +enable_traefik_provider = {{ "true" if env.get("TRAEFIK_MODE", "etcd") == "http" else "false" }} +system_timeout = 300000 +schedule_relaxed = 20 +{% elif is_standalone %} +system_timeout = 60000 +{% endif %} + +[function_proxy.args] +services_path = "/home/yuanrong/deploy/process/services.yaml" +enable_inherit_env = false +npu_collection_mode = "off" +metrics_config_file = "/home/yuanrong/metrics/metrics_config.json" +enable_direct_routing = false +force_low_reliability_instance = true +traefik_enable_tls = {{ traefik_enable_tls }} +{% if enable_metrics == "true" %} +enable_metrics = true +{% endif %} +{% if enable_trace == "true" %} +enable_trace = true +trace_config = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} +{% if is_node %} +enable_traefik_registry = {{ "true" if env.get("TRAEFIK_MODE", "etcd") == "etcd" else "false" }} +traefik_http_entrypoint = {{ toml_string(env.get("TRAEFIK_HTTP_ENTRYPOINT", "websecure")) }} +{% endif %} +log_expiration_enable = true +log_expiration_time_threshold = {{ 10 if is_agent else 7200 }} +log_expiration_cleanup_interval = {{ 10 if is_agent else 600 }} +log_expiration_max_file_count = {{ 50 if is_agent else 256 }} +{% if is_cluster %} +system_timeout = 300000 +pseudo_data_plane = true +{% elif is_node %} +system_timeout = 60000 +fc_agent_mgr_retry_times = 30 +fc_agent_mgr_retry_cycle = 60000 +runtime_logs_dir = "/home/yuanrong/logs" +{% endif %} +{% if is_agent %} +metrics_collector_type = "external" +{% endif %} + +# The CLI exposes the meta-service component port separately from values.meta_service. +[meta_service] +port = 31111 + +[iam_server.args] +token_expired_time_span = 604800 +ssl_enable = true +iam_ssl_enable = true +local_listen_port = 31113 +local_ip = "127.0.0.1" +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +enable_trace = true +{% endif %} + +{% if enable_trace == "true" and (is_cluster or is_standalone) %} +[frontend.env] +ENABLE_TRACE = "true" +TRACE_CONFIG = {{ toml_string(env["YR_TRACE_CONFIG_CONTENT"]) }} +{% endif %} diff --git a/builder/node.Dockerfile b/builder/node.Dockerfile index 9d1e173..58f322b 100644 --- a/builder/node.Dockerfile +++ b/builder/node.Dockerfile @@ -7,12 +7,12 @@ ARG AKERNEL_RUNTIME_IMAGE=akernel-runtime:local ARG AKERNEL_RUNTIME_PROFILE=rrt ARG SANDBOXD_BUILD_IMAGE=golang:1.25.5-bookworm ARG DISTILL_FS_BUILD_IMAGE=rust:1.85.0-bookworm -ARG OPEN_YR_VERSION=0.9.3 +ARG OPEN_YR_VERSION=0.9.5 ARG OPEN_YR_CORE_WHEEL_URL= ARG OPEN_YR_CORE_WHEEL_SHA256= ARG OPEN_YR_RELEASE_BASE_URL=https://github.com/openYuanrong-mirror/yuanrong/releases/download -ARG OPEN_YR_CORE_AMD64_SHA256=dd472bfa60d3d934056801ae011db7b1993cb19c5681da2395e7f1e2d84e58c3 -ARG OPEN_YR_CORE_ARM64_SHA256=4a3468d189e155e1759e2b47ace4b468d9036e76b4b750a1d47d7d13d143563e +ARG OPEN_YR_CORE_AMD64_SHA256=33157e9ab8cb0b33c49701b61c9112c3116b04df184a3769b8748c1eaa0c74b3 +ARG OPEN_YR_CORE_ARM64_SHA256=dbb8743144251b8ff8e910e41a7553cbc933bf1fd5745d58b1c34c38f612879b ARG GVISOR_RELEASE=release-20260706.0 ARG GVISOR_RELEASE_BASE_URL=https://storage.googleapis.com/gvisor/releases ARG LIBNVIDIA_CONTAINER_VERSION=1.19.1-1 @@ -20,6 +20,7 @@ ARG KATA_BUILD_IMAGE=ubuntu:24.04 ARG KATA_RELEASE=4.0.0 ARG KATA_AMD64_SHA256=2c3b9dfeba355582b40aee462b12916c9740654d0230f696adf719d67b063a8c ARG KATA_RELEASE_BASE_URL=https://github.com/kata-containers/kata-containers/releases/download +ARG KATA_LICENSE_URL=https://raw.githubusercontent.com/kata-containers/kata-containers/${KATA_RELEASE}/LICENSE ARG OTELCOL_CONTRIB_VERSION=0.120.0 ARG OTELCOL_CONTRIB_URL=https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v${OTELCOL_CONTRIB_VERSION}/otelcol-contrib_${OTELCOL_CONTRIB_VERSION}_linux_amd64.tar.gz ARG AKERNEL_VERSION=unknown @@ -29,6 +30,7 @@ FROM ${KATA_BUILD_IMAGE} AS kata-runtime ARG KATA_RELEASE ARG KATA_AMD64_SHA256 ARG KATA_RELEASE_BASE_URL +ARG KATA_LICENSE_URL ARG TARGETARCH RUN set -eux; \ test "${TARGETARCH:-amd64}" = "amd64"; \ @@ -51,8 +53,8 @@ RUN set -eux; \ ln -sfn configuration-dragonball.toml \ /kata/opt/kata/share/defaults/kata-containers/runtime-rs/configuration.toml; \ mkdir -p /kata/opt/kata/share/licenses/kata-containers; \ - curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ - "https://raw.githubusercontent.com/kata-containers/kata-containers/${KATA_RELEASE}/LICENSE" \ + curl -fSL --max-time 30 --retry 10 --retry-delay 2 --retry-all-errors \ + "${KATA_LICENSE_URL}" \ -o /kata/opt/kata/share/licenses/kata-containers/LICENSE; \ rm -f "${archive}" @@ -126,6 +128,7 @@ RUN apt-get update && \ procps \ python3 \ python3-pip \ + python3-venv \ systemd \ systemd-sysv \ tzdata \ @@ -190,6 +193,9 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ ENV YR_INSTALLATION_DIR=/home/yuanrong +ENV PATH=/opt/openyuanrong/bin:${PATH} + +COPY ./builder/config/yr/config.toml.jinja /etc/yuanrong/config.toml.jinja # Install the complete, language-runtime-free openYuanRong control plane from # its checksum-pinned core wheel. A URL and checksum pair may override the @@ -219,21 +225,23 @@ RUN set -eux; \ test -z "${OPEN_YR_CORE_WHEEL_SHA256}"; \ fi; \ wheel="/tmp/${wheel_name}"; \ - target=/tmp/openyuanrong-core; \ curl -fSL --retry 10 --retry-delay 2 --retry-all-errors \ "${wheel_url}" -o "${wheel}"; \ echo "${wheel_sha} ${wheel}" | sha256sum -c -; \ - python3 -m pip install \ - --break-system-packages \ + python3 -m venv /opt/openyuanrong; \ + /opt/openyuanrong/bin/python -m pip install \ --no-cache-dir \ - --no-deps \ - --target "${target}" \ + --index-url "${PIP_INDEX_URL}" \ "${wheel}"; \ - test -x "${target}/yr/functionsystem/bin/yr"; \ - mkdir -p "${YR_INSTALLATION_DIR}"; \ - cp -a "${target}/yr/." "${YR_INSTALLATION_DIR}/"; \ - rm -rf "${target}" "${wheel}"; \ - ln -sfn "${YR_INSTALLATION_DIR}/functionsystem/bin/yr" /usr/bin/yr + site_packages="$(/opt/openyuanrong/bin/python -c 'import site; print(site.getsitepackages()[0])')"; \ + base_py="${site_packages}/yr/cli/component/base.py"; \ + launcher_py="${site_packages}/yr/cli/system_launcher.py"; \ + sed -i \ + 's/logger.info(f"Environment: {full_env}")/logger.info(f"Environment keys: {sorted(full_env)}")/' \ + "${base_py}"; \ + sed -i 's/"env_vars": comp.env_vars,/"env_vars": {},/' "${launcher_py}"; \ + test -x /opt/openyuanrong/bin/yr; \ + rm -f "${wheel}" COPY --from=runtime-image /yr-runtime-rootfs.img ${YR_INSTALLATION_DIR}/yr-runtime-rootfs.img diff --git a/builder/runtime.Dockerfile b/builder/runtime.Dockerfile index b81100f..f7933e5 100644 --- a/builder/runtime.Dockerfile +++ b/builder/runtime.Dockerfile @@ -9,13 +9,13 @@ ARG PYTHON_311_VERSION=3.11.13 ARG PYTHON_312_VERSION=3.12.11 ARG PYTHON_313_VERSION=3.13.5 ARG PYTHON_314_VERSION=3.14.6 -ARG OPEN_YR_VERSION=0.9.3 +ARG OPEN_YR_VERSION=0.9.5 FROM ${AKERNEL_RUNTIME_BASE_IMAGE} AS rrt-download ARG OPEN_YR_VERSION ARG RRT_RUNTIME_URL=https://github.com/openYuanrong-mirror/yuanrong/releases/download/${OPEN_YR_VERSION}/rrt-runtime-amd64 -ARG RRT_RUNTIME_SHA256=89eb9271233e79f97b42b7b12cfd65e81404eb75b49d0e7a0b1ebe4977aae305 +ARG RRT_RUNTIME_SHA256=3c2b623559d9a5853b9d73e25c3e167b1b18219a1b32d95599ff9009f308d150 RUN apt-get update && \ apt-get install -y --no-install-recommends ca-certificates curl && \ diff --git a/builder/scripts/akernel-entrypoint.sh b/builder/scripts/akernel-entrypoint.sh index 1df691c..c6000ef 100644 --- a/builder/scripts/akernel-entrypoint.sh +++ b/builder/scripts/akernel-entrypoint.sh @@ -25,6 +25,16 @@ if [ -z "${role}" ]; then fi fi +case "${role}" in + master|frontend|node|standalone) + export AKERNEL_ROLE="${role}" + ;; + *) + echo "unsupported AKERNEL_ROLE: ${role}; expected master, frontend, node, or standalone" >&2 + exit 1 + ;; +esac + case "${role}" in master|frontend) /usr/local/bin/ensure-component-cert @@ -38,8 +48,4 @@ case "${role}" in /usr/local/bin/ensure-component-cert exec /usr/sbin/init "$@" ;; - *) - echo "unsupported AKERNEL_ROLE: ${role}" >&2 - exit 1 - ;; esac diff --git a/builder/scripts/master_entrypoint.sh b/builder/scripts/master_entrypoint.sh index 2a795ab..6906d3c 100644 --- a/builder/scripts/master_entrypoint.sh +++ b/builder/scripts/master_entrypoint.sh @@ -3,45 +3,93 @@ # Copyright (c) 2026 Ant Group Corporation. # # SPDX-License-Identifier: Apache-2.0 -set -e +set -euo pipefail ulimit -n 32768 -BASE_DIR=$( - cd "$(dirname "$0")" - pwd + +YR_CLI=/opt/openyuanrong/bin/yr +YR_CONFIG_TEMPLATE=/etc/yuanrong/config.toml.jinja +YR_CONFIG_PATH="${YR_RENDERED_CONFIG_PATH:-/run/yuanrong/config.toml}" +DEPLOY_PATH=/home/yuanrong/master +export DEPLOY_PATH +export YR_LOG_PATH="${DEPLOY_PATH}/log" + +if [ -z "${LITEBUS_DATA_KEY:-}" ]; then + echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 + exit 1 +fi + +if [ ! -x "${YR_CLI}" ]; then + echo "yr binary not found or not executable: ${YR_CLI}" >&2 + exit 1 +fi + +if [ "${ENABLE_TRACE:-false}" = "true" ]; then + trace_config_file="${TRACE_CONFIG_FILE:-/home/yuanrong/trace/trace_config.json}" + if [ ! -r "${trace_config_file}" ]; then + echo "trace config file is not readable: ${trace_config_file}" >&2 + exit 1 + fi + YR_TRACE_CONFIG_CONTENT="$(cat "${trace_config_file}")" + export YR_TRACE_CONFIG_CONTENT +else + unset YR_TRACE_CONFIG_CONTENT +fi + +mkdir -p "$(dirname "${YR_CONFIG_PATH}")" +"${YR_CLI}" config render \ + -t "${YR_CONFIG_TEMPLATE}" \ + -o "${YR_CONFIG_PATH}" + +YR_CLI_ARGS=( + "${YR_CLI}" + --config "${YR_CONFIG_PATH}" + start --master --block true + --port-policy FIX + --function-proxy-merge-process-enable ) -export DEPLOY_PATH="/home/yuanrong/master/" -mkdir -p "$DEPLOY_PATH" -export YR_LOG_PATH="$DEPLOY_PATH/log" -# If ConfigMap-mounted config exists, symlink it to override the baked-in default -[ -f /etc/otel-collector/otel_config.yaml ] && ln -sf /etc/otel-collector/otel_config.yaml /home/yuanrong/otel_config.yaml +if [ "${YR_CLI_DRY_RUN:-false}" = "true" ]; then + if [ -z "${YR_CLI_CAPTURE_FILE:-}" ]; then + echo "YR_CLI_CAPTURE_FILE is required when YR_CLI_DRY_RUN=true" >&2 + exit 1 + fi + mkdir -p "$(dirname "${YR_CLI_CAPTURE_FILE}")" + printf '%s\0' "${YR_CLI_ARGS[@]}" > "${YR_CLI_CAPTURE_FILE}" + exit 0 +fi + +mkdir -p "${DEPLOY_PATH}" "${YR_LOG_PATH}" -# otel watchdog: monitor and restart otelcol-contrib if it crashes +# If ConfigMap-mounted config exists, symlink it to override the baked-in default. +[ -f /etc/otel-collector/otel_config.yaml ] && + ln -sf /etc/otel-collector/otel_config.yaml /home/yuanrong/otel_config.yaml + +# Monitor and restart the collector when observability is enabled. otel_watchdog() { - local otel_log="$DEPLOY_PATH/otelcol.log" + local otel_log="${DEPLOY_PATH}/otelcol.log" local max_restart_interval=60 local restart_count=0 while true; do - otelcol-contrib --config="/home/yuanrong/otel_config.yaml" >> "$otel_log" 2>&1 & + otelcol-contrib --config=/home/yuanrong/otel_config.yaml >> "${otel_log}" 2>&1 & local otel_pid=$! - echo $otel_pid > $DEPLOY_PATH/otelcol.pid - echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol started, PID: $otel_pid (restart count: $restart_count)" >> "$otel_log" + echo "${otel_pid}" > "${DEPLOY_PATH}/otelcol.pid" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol started, PID: ${otel_pid} (restart count: ${restart_count})" >> "${otel_log}" - wait $otel_pid - local exit_code=$? + local exit_code=0 + wait "${otel_pid}" || exit_code=$? restart_count=$((restart_count + 1)) - echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol exited with code $exit_code, restarting in 5s (restart count: $restart_count)" >> "$otel_log" + echo "[$(date '+%Y-%m-%d %H:%M:%S')] otelcol exited with code ${exit_code}, restarting (restart count: ${restart_count})" >> "${otel_log}" - # Exponential backoff with a cap local delay=$((2 ** restart_count)) - [ $delay -gt $max_restart_interval ] && delay=$max_restart_interval - sleep $delay + [ "${delay}" -gt "${max_restart_interval}" ] && delay="${max_restart_interval}" + sleep "${delay}" done } export -f otel_watchdog -if { [ "${ENABLE_METRICS:-false}" = "true" ] || [ "${ENABLE_TRACE:-false}" = "true" ]; } && command -v otelcol-contrib >/dev/null 2>&1; then +if { [ "${ENABLE_METRICS:-false}" = "true" ] || [ "${ENABLE_TRACE:-false}" = "true" ]; } && + command -v otelcol-contrib >/dev/null 2>&1; then nohup bash -c otel_watchdog & echo "otelcol watchdog started" echo "otel log: ${DEPLOY_PATH}/otelcol.log" @@ -49,64 +97,4 @@ else echo "otelcol watchdog skipped" fi -# Set enable_traefik_provider based on TRAEFIK_MODE -if [ "${TRAEFIK_MODE:-etcd}" = "http" ]; then - ENABLE_TRAEFIK_PROVIDER=true -else - ENABLE_TRAEFIK_PROVIDER=false -fi - -if [ -z "${LITEBUS_DATA_KEY:-}" ]; then - echo "LITEBUS_DATA_KEY is required for akernel master/frontend" >&2 - exit 1 -fi - -YR_BIN="${YR_BIN:-/usr/bin/yr}" -if [ ! -x "${YR_BIN}" ]; then - echo "yr binary not found or not executable: ${YR_BIN}" >&2 - exit 1 -fi - -exec "${YR_BIN}" start --master --block true \ - -e -c 0 -m 8000 -s 4096 -n $HOSTNAME \ - -d $DEPLOY_PATH \ - --fs_health_check_retry_interval 1 \ - --schedule_relaxed 20 \ - --enable_faas_frontend ${ENABLE_FAAS_FRONTEND:-true} \ - --enable_function_scheduler ${ENABLE_FUNCTION_SCHEDULER:-false} \ - --enable_meta_service ${ENABLE_META_SERVICE:-true} \ - --enable_iam_server ${ENABLE_IAM_SERVER:-true} \ - --iam_token_expired_time_span 604800 \ - --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ - --frontend_client_auth_type NoClientCert \ - --enable_function_token_auth ${ENABLE_FUNCTION_TOKEN_AUTH:-true} \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --port_policy FIX \ - --system_timeout 300000 \ - --enable_distributed_master false \ - --etcd_mode outter \ - --etcd_addr_list $ETCD_ADDRESS \ - --etcd_port ${ETCD_PORT} \ - --etcd_peer_port 2378 \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - --ds_rpc_thread_num 128 \ - --function_proxy_merge_process_enable true \ - --enable_traefik_provider=${ENABLE_TRAEFIK_PROVIDER} \ - --traefik_http_entry_point=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_forward_timeout_ms=3000 \ - --frontend_lease_bypass true \ - --iam_ssl_enable true \ - --ssl_root_file ca.crt \ - --ssl_cert_file module.crt \ - --ssl_key_file module.key \ - --iam_local_listen_port 31113 \ - --iam_local_ip 127.0.0.1 \ - --enable_direct_routing false \ - --enable_sandbox_router true \ - ${META_SERVICE_ADDRESS:+--meta_service_address $META_SERVICE_ADDRESS} +exec "${YR_CLI_ARGS[@]}" diff --git a/builder/scripts/yr_node_bootstrap.sh b/builder/scripts/yr_node_bootstrap.sh index c4d5ae5..944ec66 100755 --- a/builder/scripts/yr_node_bootstrap.sh +++ b/builder/scripts/yr_node_bootstrap.sh @@ -3,7 +3,13 @@ # Copyright (c) 2026 Ant Group Corporation. # # SPDX-License-Identifier: Apache-2.0 +set -euo pipefail + ulimit -n 32768 + +YR_CLI=/opt/openyuanrong/bin/yr +YR_CONFIG_TEMPLATE=/etc/yuanrong/config.toml.jinja +YR_CONFIG_PATH="${YR_RENDERED_CONFIG_PATH:-/run/yuanrong/config.toml}" export YR_RUNTIME_BACKEND=sandboxd resolve_node_ip() { @@ -39,107 +45,112 @@ resolve_node_ip() { printf '%s\n' "${node_ip}" } +wait_for_sandbox_ip() { + local address + local attempt + + for ((attempt = 0; attempt < 60; attempt++)); do + address="$( + ip -4 -o address show dev sandbox0 scope global 2>/dev/null | + awk 'NR == 1 { split($4, value, "/"); print value[1] }' + )" || true + if [ -n "${address}" ]; then + printf '%s\n' "${address}" + return + fi + sleep 1 + done + + echo "timed out after 60s waiting for sandbox0 to have an IPv4 address" >&2 + return 1 +} + YR_NODE_IP="$(resolve_node_ip)" +export YR_NODE_IP echo "Using ${YR_NODE_IP} as the YuanRong node address" -# Set enable_traefik_registry based on TRAEFIK_MODE -if [ "${TRAEFIK_MODE:-etcd}" = "etcd" ]; then - ENABLE_TRAEFIK_REGISTRY=${ENABLE_TRAEFIK_REGISTRY:-true} -else - ENABLE_TRAEFIK_REGISTRY=false -fi +YR_LOCAL_IP="$(wait_for_sandbox_ip)" +export YR_LOCAL_IP +echo "Using ${YR_LOCAL_IP} as the YuanRong sandbox address" -if [ "x${AKS_LOCAL_MODE}" == "xtrue" ]; then - if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then - LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" - export LITEBUS_DATA_KEY +role="${AKERNEL_ROLE:-}" +if [ -z "${role}" ]; then + if [ "${AKS_LOCAL_MODE:-false}" = "true" ]; then + role=standalone + else + role=node fi - if [ -z "${LITEBUS_DATA_KEY:-}" ]; then - echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 + export AKERNEL_ROLE="${role}" +fi + +case "${role}" in + node) + ;; + standalone) + if [ -z "${LITEBUS_DATA_KEY:-}" ] && [ -r /home/akernel/iam-seed ]; then + LITEBUS_DATA_KEY="$(tr -d '[:space:]' < /home/akernel/iam-seed)" + export LITEBUS_DATA_KEY + fi + if [ -z "${LITEBUS_DATA_KEY:-}" ]; then + echo "LITEBUS_DATA_KEY is required in standalone mode" >&2 + exit 1 + fi + ;; + *) + echo "AKERNEL_ROLE must be node or standalone" >&2 + exit 1 + ;; +esac + +if [ ! -f "${YR_CONFIG_TEMPLATE}" ]; then + echo "YuanRong CLI config template not found: ${YR_CONFIG_TEMPLATE}" >&2 + exit 1 +fi +if [ ! -x "${YR_CLI}" ]; then + echo "YuanRong CLI not executable: ${YR_CLI}" >&2 + exit 1 +fi + +if [ "${ENABLE_TRACE:-false}" = "true" ]; then + trace_config_file="${TRACE_CONFIG_FILE:-/home/yuanrong/trace/trace_config.json}" + if [ ! -r "${trace_config_file}" ]; then + echo "trace config file is not readable: ${trace_config_file}" >&2 exit 1 fi - /usr/bin/yr start --master \ - --ip_address "${YR_NODE_IP}" \ - --port_policy FIX \ - --enable_function_scheduler=false \ - --enable_faas_frontend=true \ - --enable_meta_service=true \ - --enable_iam_server=true \ - --iam_token_expired_time_span 604800 \ - --ssl_base_path=/home/yuanrong/.cert/ \ - --frontend_ssl_enable=true \ - --frontend_client_auth_type NoClientCert \ - --enable_function_token_auth true \ - --ds_node_timeout_s 30 \ - --ds_client_dead_timeout_s 60 \ - --ds_heartbeat_interval_ms 1000 \ - --ds_node_dead_timeout_s 120 \ - --system_timeout 60000 \ - --block true \ - --etcd_port ${ETCD_PORT:-2379} \ - --etcd_peer_port ${ETCD_PEER_PORT:-2378} \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --enable_distributed_master false \ - --metrics_collector_type external \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - --log_root "${YR_LOG_PATH}" \ - --function_proxy_merge_process_enable true \ - --fc_agent_mgr_retry_times 30 \ - --fc_agent_mgr_retry_cycle 60000 \ - --iam_ssl_enable true \ - --ssl_root_file ca.crt \ - --ssl_cert_file module.crt \ - --ssl_key_file module.key \ - --iam_local_listen_port 31113 \ - --iam_local_ip 127.0.0.1 \ - --frontend_lease_bypass true \ - --force_low_reliability_instance true \ - --enable_sandbox_router true \ - --enable_direct_routing false + YR_TRACE_CONFIG_CONTENT="$(cat "${trace_config_file}")" + export YR_TRACE_CONFIG_CONTENT else - /usr/bin/yr start \ - --ip_address "${YR_NODE_IP}" \ - --port_policy FIX \ - --ds_node_timeout_s 30 \ - --ds_client_dead_timeout_s 60 \ - --ds_heartbeat_interval_ms 1000 \ - --ds_node_dead_timeout_s 120 \ - --etcd_addr_list ${ETCD_ADDRESS} \ - --etcd_mode outter \ - --etcd_port ${ETCD_PORT} \ - --etcd_peer_port ${ETCD_PEER_PORT:-2378} \ - --system_timeout 60000 \ - --enable_inherit_env false \ - --npu_collection_mode off \ - --enable_distributed_master false \ - --metrics_collector_type external \ - --enable_metrics ${ENABLE_METRICS} \ - --metrics_config_file "/home/yuanrong/metrics/metrics_config.json" \ - --enable_trace ${ENABLE_TRACE} \ - --trace_config "$(cat /home/yuanrong/trace/trace_config.json)" \ - -n ${HOSTNAME} \ - --enable_traefik_registry=${ENABLE_TRAEFIK_REGISTRY} \ - --traefik_enable_tls=${TRAEFIK_ENABLE_TLS:-false} \ - --traefik_etcd_prefix=traefik \ - --traefik_lease_ttl=300000 \ - --traefik_http_entrypoint=${TRAEFIK_HTTP_ENTRYPOINT:-websecure} \ - --log_root "${YR_LOG_PATH}" \ - --fc_agent_mgr_retry_times 30 \ - --fc_agent_mgr_retry_cycle 60000 \ - --log_expiration_time_threshold 10 \ - --log_expiration_cleanup_interval 10 \ - --log_expiration_max_file_count 50 \ - --function_proxy_merge_process_enable true \ - --enable_direct_routing false \ - --force_low_reliability_instance true \ - --block true + unset YR_TRACE_CONFIG_CONTENT +fi + +mkdir -p "$(dirname "${YR_CONFIG_PATH}")" +"${YR_CLI}" config render \ + -t "${YR_CONFIG_TEMPLATE}" \ + -o "${YR_CONFIG_PATH}" + +YR_CLI_ARGS=( + "${YR_CLI}" + --config "${YR_CONFIG_PATH}" + start +) +if [ "${role}" = "standalone" ]; then + YR_CLI_ARGS+=(--master) fi +YR_CLI_ARGS+=( + --block true + --port-policy FIX + --function-proxy-merge-process-enable +) + +if [ "${YR_CLI_DRY_RUN:-false}" = "true" ]; then + if [ -z "${YR_CLI_CAPTURE_FILE:-}" ]; then + echo "YR_CLI_CAPTURE_FILE is required when YR_CLI_DRY_RUN=true" >&2 + exit 1 + fi + mkdir -p "$(dirname "${YR_CLI_CAPTURE_FILE}")" + printf '%s\0' "${YR_CLI_ARGS[@]}" > "${YR_CLI_CAPTURE_FILE}" + exit 0 +fi + +mkdir -p "${YR_LOG_PATH:-/home/yuanrong/logs}" +exec "${YR_CLI_ARGS[@]}" diff --git a/builder/systemd_services/yuanrong.service b/builder/systemd_services/yuanrong.service index 3960526..621c6bb 100644 --- a/builder/systemd_services/yuanrong.service +++ b/builder/systemd_services/yuanrong.service @@ -4,7 +4,7 @@ Description=yuanrong.service [Service] #Type=simple PIDFile=/run/yuanrong.pid -PassEnvironment=ETCD_PORT ETCD_PEER_PORT ETCD_ADDRESS HOSTNAME AKS_LOCAL_MODE AKERNEL_NODE_IP INSTANCE_IP LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT +PassEnvironment=AKERNEL_ROLE AKERNEL_NODE_IP INSTANCE_IP ETCD_ADDRESS ETCD_PORT ETCD_PEER_PORT HOSTNAME AKS_LOCAL_MODE LITEBUS_DATA_KEY YR_LOG_PATH YR_INSTALLATION_DIR ENABLE_METRICS ENABLE_TRACE TRACE_CONFIG_FILE TRAEFIK_MODE TRAEFIK_ENABLE_TLS TRAEFIK_HTTP_ENTRYPOINT Environment="CONTAINER_EP=unix:///run/sandboxd/sandboxd.sock" Environment="RUNTIME_HOME_DIR=/home/yuanrong/runtime" Environment="YR_NOSET_CUDA_VISIBLE_DEVICES=1" @@ -14,6 +14,7 @@ ExecStart=/usr/bin/bash /home/yuanrong/yr_node_bootstrap.sh ExecReload=/bin/kill -15 $MAINPID KillMode=control-group Restart=always +RestartSec=5s Delegate=yes UMask=000 TasksMax=infinity diff --git a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml index 2b02967..1645d45 100644 --- a/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml +++ b/deploy/akernel/charts/core/templates/frontend/akernel_frontend.yaml @@ -94,18 +94,8 @@ spec: secretKeyRef: name: {{ include "core.litebusSecretName" . }} key: litebus-data-key - - name: ENABLE_FUNCTION_MASTER - value: "false" - - name: ENABLE_FUNCTION_SCHEDULER - value: "false" - - name: ENABLE_META_SERVICE - value: "false" - - name: ENABLE_IAM_SERVER - value: "true" - name: META_SERVICE_ADDRESS value: {{ .Values.frontend.master.metaServiceAddress | default (printf "akernel-master.%s.svc.cluster.local:31111" .Release.Namespace) | quote }} - - name: IAM_SERVER_ADDRESS - value: {{ .Values.frontend.master.iamServerAddress | default (printf "akernel-master.%s.svc.cluster.local:31112" .Release.Namespace) | quote }} - name: ETCD_ADDRESS value: {{ get $frontendEtcd "host" | default (printf "akernel-etcd.%s.svc.cluster.local" .Release.Namespace) | quote }} - name: ETCD_PORT @@ -124,10 +114,14 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} ports: - name: http containerPort: 8888 diff --git a/deploy/akernel/charts/core/templates/master/akernel_master.yaml b/deploy/akernel/charts/core/templates/master/akernel_master.yaml index de37591..0e1957f 100644 --- a/deploy/akernel/charts/core/templates/master/akernel_master.yaml +++ b/deploy/akernel/charts/core/templates/master/akernel_master.yaml @@ -108,16 +108,24 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} - name: TRAEFIK_MODE value: {{ .Values.traefik.mode | default "http" | quote }} + {{- if and (not (.Values.traefik.enableWebEntrypoint | default false)) (.Values.traefik.enableTLS | default false) }} - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} + value: "true" + {{- end }} + {{- if .Values.traefik.enableWebEntrypoint }} - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} + value: "web" + {{- end }} ports: - name: http containerPort: 8888 diff --git a/deploy/akernel/charts/core/templates/node/configmap.yaml b/deploy/akernel/charts/core/templates/node/configmap.yaml index ec936d7..f9bcc82 100644 --- a/deploy/akernel/charts/core/templates/node/configmap.yaml +++ b/deploy/akernel/charts/core/templates/node/configmap.yaml @@ -13,7 +13,7 @@ data: CPUQuota={{ .Values.node.config.resourceControl.cpuQuota | default "800%" }} sandboxd_config.toml: | -{{ .Values.node.config.sandboxd.config | indent 4 }} +{{ tpl .Values.node.config.sandboxd.config . | indent 4 }} registry.json: | {{ .Values.node.config.registry | toPrettyJson | indent 4 }} diff --git a/deploy/akernel/charts/core/templates/node/daemonset.yaml b/deploy/akernel/charts/core/templates/node/daemonset.yaml index 36b8443..8a9f305 100644 --- a/deploy/akernel/charts/core/templates/node/daemonset.yaml +++ b/deploy/akernel/charts/core/templates/node/daemonset.yaml @@ -118,16 +118,24 @@ spec: {{- end }} - name: AKERNEL_ENV value: {{ .Values.monitoring.akernelEnv | default "default" | quote }} + {{- if .Values.monitoring.prometheusEndpoint }} - name: ENABLE_METRICS - value: {{ ne (.Values.monitoring.prometheusEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} + {{- if .Values.monitoring.tempoEndpoint }} - name: ENABLE_TRACE - value: {{ ne (.Values.monitoring.tempoEndpoint | default "") "" | ternary "true" "false" | quote }} + value: "true" + {{- end }} - name: TRAEFIK_MODE value: {{ .Values.traefik.mode | default "http" | quote }} + {{- if and (not (.Values.traefik.enableWebEntrypoint | default false)) (.Values.traefik.enableTLS | default false) }} - name: TRAEFIK_ENABLE_TLS - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary false (.Values.traefik.enableTLS | default false) | quote }} + value: "true" + {{- end }} + {{- if .Values.traefik.enableWebEntrypoint }} - name: TRAEFIK_HTTP_ENTRYPOINT - value: {{ .Values.traefik.enableWebEntrypoint | default false | ternary "web" "websecure" | quote }} + value: "web" + {{- end }} securityContext: privileged: true volumeMounts: diff --git a/deploy/akernel/charts/core/values.yaml b/deploy/akernel/charts/core/values.yaml index 20d9982..e0ecee5 100644 --- a/deploy/akernel/charts/core/values.yaml +++ b/deploy/akernel/charts/core/values.yaml @@ -120,6 +120,11 @@ frontend: iamServerAddress: "" node: + # sandboxd allocates sandbox addresses from this CIDR. + sandboxIPRange: "172.17.0.1/16" + etcd: + host: "" + port: "2379" secret: create: true oss_auths: {} @@ -462,7 +467,7 @@ node: stream_server_port="" [plugin.network] - ip_range="172.17.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="iptables" [plugin.resource] diff --git a/deploy/standalone/README.md b/deploy/standalone/README.md index 35d42a1..abc387b 100644 --- a/deploy/standalone/README.md +++ b/deploy/standalone/README.md @@ -124,6 +124,9 @@ This will: default) - Print the Traefik container IP to use as `AKERNEL_SERVER_ADDRESS` +The openYuanRong bootstrap waits for `sandbox0` and uses the IPv4 address +actually assigned to that interface as `yr start --local_ip`. + No host ports are published. On Linux, the host accesses Traefik directly through its Docker bridge IP. diff --git a/deploy/terraform/aliyun/values-akernel.yaml.tmpl b/deploy/terraform/aliyun/values-akernel.yaml.tmpl index 474fb19..64c7de8 100644 --- a/deploy/terraform/aliyun/values-akernel.yaml.tmpl +++ b/deploy/terraform/aliyun/values-akernel.yaml.tmpl @@ -77,6 +77,7 @@ frontend: %{ endif ~} node: + sandboxIPRange: "172.17.0.1/16" affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -177,7 +178,7 @@ node: stream_server_port="" [plugin.network] - ip_range="172.17.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="${sandboxd_nat_backend}" [plugin.resource] diff --git a/deploy/terraform/huaweicloud/main.tf b/deploy/terraform/huaweicloud/main.tf index 99cdc02..9810fd4 100644 --- a/deploy/terraform/huaweicloud/main.tf +++ b/deploy/terraform/huaweicloud/main.tf @@ -39,8 +39,10 @@ locals { } : {} oss_auths = merge(local.generated_oss_auths, var.oss_auths) registry_auths = { - auths = { for host, cred in var.registry_auths : host => { username = cred.username, password = cred.password } } + auths = { for host, cred in var.registry_auths : host => { auth = base64encode("${cred.username}:${cred.password}") } } } + registry_auths_enabled = length(var.registry_auths) > 0 + dockerconfigjson = base64encode(jsonencode(local.registry_auths)) # When auto-creating ELB on Huawei Cloud CCE, inject required annotations # so the cloud-controller-manager provisions the ELB automatically. @@ -118,6 +120,8 @@ locals { node_home_csi_size = var.node_home_csi_size oss_auths = local.oss_auths registry_auths = local.registry_auths + registry_auths_enabled = local.registry_auths_enabled + dockerconfigjson = local.dockerconfigjson etcd_cpu = var.etcd_resources.cpu etcd_memory = var.etcd_resources.memory diff --git a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl index 1d8622a..25ee0b1 100644 --- a/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl +++ b/deploy/terraform/huaweicloud/values-akernel.yaml.tmpl @@ -1,3 +1,13 @@ +%{ if registry_auths_enabled ~} +imagePullSecrets: + - name: registry-secret + +imagePullSecret: + create: true + name: "registry-secret" + dockerconfigjson: "${dockerconfigjson}" +%{ endif ~} + kruise: enabled: ${enable_kruise} @@ -68,6 +78,7 @@ frontend: %{ endif ~} node: + sandboxIPRange: "172.17.0.1/16" image: repository: "${node_image_repository}" tag: "${node_image_tag}" @@ -135,7 +146,7 @@ node: stream_server_port="" [plugin.network] - ip_range="172.17.0.1/16" + ip_range="{{ .Values.node.sandboxIPRange }}" nat_backend="${sandboxd_nat_backend}" [plugin.resource] diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index b4d284f..571dea5 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -28,21 +28,21 @@ classifiers = [ "Topic :: System :: Distributed Computing", ] dependencies = [ - "openyuanrong-sandbox==0.9.3", + "openyuanrong-sandbox==0.9.5", "websockets>=10.0", ] [project.optional-dependencies] openyuanrong-sdk = [ - "openyuanrong-sdk==0.9.3", + "openyuanrong-sdk==0.9.5", ] all = [ - "openyuanrong-sdk==0.9.3", + "openyuanrong-sdk==0.9.5", ] dev = [ "build>=1.2,<2", "mypy>=1.10,<2", - "openyuanrong-sdk==0.9.3", + "openyuanrong-sdk==0.9.5", "ruff>=0.11,<1", ]