From f9982031b8b4a433d759e9c6a0b79f75c95651ec Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 20 May 2026 10:18:27 +0800 Subject: [PATCH 1/7] Update metrics labels and add aproxy log forwarding --- .../templates/pre-job.j2 | 88 ++++++++++++++++--- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 index c7d5f9b9b..23e4de83c 100644 --- a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 +++ b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 @@ -144,29 +144,82 @@ receivers: collection_interval: 10s scrapers: cpu: + metrics: + system.cpu.logical.count: + enabled: true + system.cpu.physical.count: + enabled: true memory: disk: filesystem: network: load: + journald/self_hosted_runner_aproxy: + units: + - snap.aproxy.aproxy.service processors: - attributes/github_labels: - actions: - - key: github_runner + resource/self_hosted_runner_github_labels: + attributes: + - key: service.name action: upsert - value: "$RUNNER_NAME" - - key: github_workflow + value: "self-hosted-runner" + - key: github.repository action: upsert - value: "$GITHUB_WORKFLOW" - - key: github_job + value: "$GITHUB_REPOSITORY" + - key: github.runner action: upsert - value: "$GITHUB_JOB" - - key: github_repository + value: "$RUNNER_NAME" + - key: github.workflow action: upsert - value: "$GITHUB_REPOSITORY" + value: "$GITHUB_WORKFLOW" + - key: github.job + action: upsert + value: "$GITHUB_JOB" + - key: github.run.id + action: upsert + value: "$GITHUB_RUN_ID" + - key: github.run.attempt + action: upsert + value: "$GITHUB_RUN_ATTEMPT" + resource/self_hosted_runner_journald_attributes: + attributes: + - key: systemd.unit + from_attribute: systemd.unit + action: upsert + transform/self_hosted_runner_journald_loki_labels: + log_statements: + - context: resource + statements: + - > + set(attributes["loki.resource.labels"], + Concat([attributes["loki.resource.labels"], ", systemd.unit"], "")) + where attributes["loki.resource.labels"] != nil + - > + set(attributes["loki.resource.labels"], "systemd.unit") + where attributes["loki.resource.labels"] == nil + transform/self_hosted_runner_loki_labels: + log_statements: + - context: resource + statements: + - > + set(attributes["loki.resource.labels"], + Concat([attributes["loki.resource.labels"], + ", service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt"], "")) + where attributes["loki.resource.labels"] != nil + - > + set(attributes["loki.resource.labels"], + "service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt") + where attributes["loki.resource.labels"] == nil + - > + set(attributes["loki.attribute.labels"], + Concat([attributes["loki.attribute.labels"], ", service.name"], "")) + where attributes["loki.attribute.labels"] != nil + - > + set(attributes["loki.attribute.labels"], "service.name") + where attributes["loki.attribute.labels"] == nil batch: exporters: - otlp/mimir: + otlp/self_hosted_runner: endpoint: {{ otel_collector_endpoint }} tls: insecure: true @@ -174,8 +227,17 @@ service: pipelines: metrics: receivers: [hostmetrics] - processors: [attributes/github_labels, batch] - exporters: [otlp/mimir] + processors: [resource/self_hosted_runner_github_labels, batch] + exporters: [otlp/self_hosted_runner] + logs: + receivers: [journald/self_hosted_runner_aproxy] + processors: + - resource/self_hosted_runner_journald_attributes + - transform/self_hosted_runner_journald_loki_labels + - resource/self_hosted_runner_github_labels + - transform/self_hosted_runner_loki_labels + - batch + exporters: [otlp/self_hosted_runner] EOF /usr/bin/sudo /usr/bin/snap enable opentelemetry-collector From e4c305a3a41a910e31905cba1736b86369a7823e Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 20 May 2026 10:19:38 +0800 Subject: [PATCH 2/7] Update metrics labels and add aproxy log forwarding --- .../src/github_runner_manager/templates/pre-job.j2 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 index 23e4de83c..7669d9da7 100644 --- a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 +++ b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 @@ -210,13 +210,6 @@ processors: set(attributes["loki.resource.labels"], "service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt") where attributes["loki.resource.labels"] == nil - - > - set(attributes["loki.attribute.labels"], - Concat([attributes["loki.attribute.labels"], ", service.name"], "")) - where attributes["loki.attribute.labels"] != nil - - > - set(attributes["loki.attribute.labels"], "service.name") - where attributes["loki.attribute.labels"] == nil batch: exporters: otlp/self_hosted_runner: From 69946c601a5a18c0b7097f73ca1bda3693a030f6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 20 May 2026 10:34:07 +0800 Subject: [PATCH 3/7] Add local listeners --- .../templates/pre-job.j2 | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 index 7669d9da7..0e45eebca 100644 --- a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 +++ b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 @@ -157,6 +157,21 @@ receivers: journald/self_hosted_runner_aproxy: units: - snap.aproxy.aproxy.service + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:44317 + http: + endpoint: 0.0.0.0:44318 + prometheusremotewrite: + endpoint: 0.0.0.0:49090 + default_external_labels: + source: github-runner + loki: + protocols: + http: + endpoint: 0.0.0.0:43100 + use_incoming_timestamp: true processors: resource/self_hosted_runner_github_labels: attributes: @@ -222,6 +237,14 @@ service: receivers: [hostmetrics] processors: [resource/self_hosted_runner_github_labels, batch] exporters: [otlp/self_hosted_runner] + metrics/otlp: + receivers: [otlp] + processors: [resource/self_hosted_runner_github_labels, batch] + exporters: [otlp/self_hosted_runner] + metrics/prometheus: + receivers: [prometheusremotewrite] + processors: [resource/self_hosted_runner_github_labels, batch] + exporters: [otlp/self_hosted_runner] logs: receivers: [journald/self_hosted_runner_aproxy] processors: @@ -231,6 +254,24 @@ service: - transform/self_hosted_runner_loki_labels - batch exporters: [otlp/self_hosted_runner] + logs/otlp: + receivers: [otlp] + processors: + - resource/self_hosted_runner_github_labels + - transform/self_hosted_runner_loki_labels + - batch + exporters: [otlp/self_hosted_runner] + logs/loki: + receivers: [loki] + processors: + - resource/self_hosted_runner_github_labels + - transform/self_hosted_runner_loki_labels + - batch + exporters: [otlp/self_hosted_runner] + traces: + receivers: [otlp] + processors: [resource/self_hosted_runner_github_labels, batch] + exporters: [otlp/self_hosted_runner] EOF /usr/bin/sudo /usr/bin/snap enable opentelemetry-collector From 34ca18788302eba2622816436a7220b9337704a2 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 20 May 2026 13:18:59 +0800 Subject: [PATCH 4/7] Add local listeners --- .../templates/pre-job.j2 | 48 +++---------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 index 0e45eebca..bd7026e79 100644 --- a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 +++ b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 @@ -154,19 +154,12 @@ receivers: filesystem: network: load: - journald/self_hosted_runner_aproxy: - units: - - snap.aproxy.aproxy.service otlp: protocols: grpc: endpoint: 0.0.0.0:44317 http: endpoint: 0.0.0.0:44318 - prometheusremotewrite: - endpoint: 0.0.0.0:49090 - default_external_labels: - source: github-runner loki: protocols: http: @@ -196,39 +189,23 @@ processors: - key: github.run.attempt action: upsert value: "$GITHUB_RUN_ATTEMPT" - resource/self_hosted_runner_journald_attributes: - attributes: - - key: systemd.unit - from_attribute: systemd.unit - action: upsert - transform/self_hosted_runner_journald_loki_labels: - log_statements: - - context: resource - statements: - - > - set(attributes["loki.resource.labels"], - Concat([attributes["loki.resource.labels"], ", systemd.unit"], "")) - where attributes["loki.resource.labels"] != nil - - > - set(attributes["loki.resource.labels"], "systemd.unit") - where attributes["loki.resource.labels"] == nil transform/self_hosted_runner_loki_labels: log_statements: - context: resource statements: - > - set(attributes["loki.resource.labels"], - Concat([attributes["loki.resource.labels"], + set(resource.attributes["loki.resource.labels"], + Concat([resource.attributes["loki.resource.labels"], ", service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt"], "")) - where attributes["loki.resource.labels"] != nil + where resource.attributes["loki.resource.labels"] != nil - > - set(attributes["loki.resource.labels"], + set(resource.attributes["loki.resource.labels"], "service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt") - where attributes["loki.resource.labels"] == nil + where resource.attributes["loki.resource.labels"] == nil batch: exporters: otlp/self_hosted_runner: - endpoint: {{ otel_collector_endpoint }} + endpoint: "$ACTION_OTEL_EXPORTER_OTLP_ENDPOINT" tls: insecure: true service: @@ -241,19 +218,6 @@ service: receivers: [otlp] processors: [resource/self_hosted_runner_github_labels, batch] exporters: [otlp/self_hosted_runner] - metrics/prometheus: - receivers: [prometheusremotewrite] - processors: [resource/self_hosted_runner_github_labels, batch] - exporters: [otlp/self_hosted_runner] - logs: - receivers: [journald/self_hosted_runner_aproxy] - processors: - - resource/self_hosted_runner_journald_attributes - - transform/self_hosted_runner_journald_loki_labels - - resource/self_hosted_runner_github_labels - - transform/self_hosted_runner_loki_labels - - batch - exporters: [otlp/self_hosted_runner] logs/otlp: receivers: [otlp] processors: From 5ec616932adaf7a799d28a83dad07fdff061c309 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 20 May 2026 13:36:32 +0800 Subject: [PATCH 5/7] Add local listeners --- .../templates/pre-job.j2 | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 index bd7026e79..9b7e27510 100644 --- a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 +++ b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 @@ -166,8 +166,8 @@ receivers: endpoint: 0.0.0.0:43100 use_incoming_timestamp: true processors: - resource/self_hosted_runner_github_labels: - attributes: + attributes/self_hosted_runner_github_labels: + actions: - key: service.name action: upsert value: "self-hosted-runner" @@ -189,6 +189,21 @@ processors: - key: github.run.attempt action: upsert value: "$GITHUB_RUN_ATTEMPT" + - key: host.arch + action: upsert + value: "$(uname -m)" + transform/self_hosted_runner_attributes_to_resource: + log_statements: + - context: log + statements: + - set(resource.attributes["service.name"], attributes["service.name"]) where attributes["service.name"] != nil + - set(resource.attributes["github.repository"], attributes["github.repository"]) where attributes["github.repository"] != nil + - set(resource.attributes["github.runner"], attributes["github.runner"]) where attributes["github.runner"] != nil + - set(resource.attributes["github.workflow"], attributes["github.workflow"]) where attributes["github.workflow"] != nil + - set(resource.attributes["github.job"], attributes["github.job"]) where attributes["github.job"] != nil + - set(resource.attributes["github.run.id"], attributes["github.run.id"]) where attributes["github.run.id"] != nil + - set(resource.attributes["github.run.attempt"], attributes["github.run.attempt"]) where attributes["github.run.attempt"] != nil + - set(resource.attributes["host.arch"], attributes["host.arch"]) where attributes["host.arch"] != nil transform/self_hosted_runner_loki_labels: log_statements: - context: resource @@ -196,11 +211,11 @@ processors: - > set(resource.attributes["loki.resource.labels"], Concat([resource.attributes["loki.resource.labels"], - ", service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt"], "")) + ", service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt, host.arch"], "")) where resource.attributes["loki.resource.labels"] != nil - > set(resource.attributes["loki.resource.labels"], - "service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt") + "service.name, github.repository, github.runner, github.workflow, github.job, github.run.id, github.run.attempt, host.arch") where resource.attributes["loki.resource.labels"] == nil batch: exporters: @@ -212,29 +227,31 @@ service: pipelines: metrics: receivers: [hostmetrics] - processors: [resource/self_hosted_runner_github_labels, batch] + processors: [attributes/self_hosted_runner_github_labels, batch] exporters: [otlp/self_hosted_runner] metrics/otlp: receivers: [otlp] - processors: [resource/self_hosted_runner_github_labels, batch] + processors: [attributes/self_hosted_runner_github_labels, batch] exporters: [otlp/self_hosted_runner] logs/otlp: receivers: [otlp] processors: - - resource/self_hosted_runner_github_labels + - attributes/self_hosted_runner_github_labels + - transform/self_hosted_runner_attributes_to_resource - transform/self_hosted_runner_loki_labels - batch exporters: [otlp/self_hosted_runner] logs/loki: receivers: [loki] processors: - - resource/self_hosted_runner_github_labels + - attributes/self_hosted_runner_github_labels + - transform/self_hosted_runner_attributes_to_resource - transform/self_hosted_runner_loki_labels - batch exporters: [otlp/self_hosted_runner] traces: receivers: [otlp] - processors: [resource/self_hosted_runner_github_labels, batch] + processors: [attributes/self_hosted_runner_github_labels, batch] exporters: [otlp/self_hosted_runner] EOF From 8244790bc04ebb4dbb4b9ac8da713372003113b6 Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 20 May 2026 13:45:05 +0800 Subject: [PATCH 6/7] Update --- .../templates/pre-job.j2 | 46 ++++++++++++------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 index 9b7e27510..3e6d307e6 100644 --- a/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 +++ b/github-runner-manager/src/github_runner_manager/templates/pre-job.j2 @@ -192,18 +192,32 @@ processors: - key: host.arch action: upsert value: "$(uname -m)" - transform/self_hosted_runner_attributes_to_resource: - log_statements: - - context: log - statements: - - set(resource.attributes["service.name"], attributes["service.name"]) where attributes["service.name"] != nil - - set(resource.attributes["github.repository"], attributes["github.repository"]) where attributes["github.repository"] != nil - - set(resource.attributes["github.runner"], attributes["github.runner"]) where attributes["github.runner"] != nil - - set(resource.attributes["github.workflow"], attributes["github.workflow"]) where attributes["github.workflow"] != nil - - set(resource.attributes["github.job"], attributes["github.job"]) where attributes["github.job"] != nil - - set(resource.attributes["github.run.id"], attributes["github.run.id"]) where attributes["github.run.id"] != nil - - set(resource.attributes["github.run.attempt"], attributes["github.run.attempt"]) where attributes["github.run.attempt"] != nil - - set(resource.attributes["host.arch"], attributes["host.arch"]) where attributes["host.arch"] != nil + resource/self_hosted_runner_github_labels: + attributes: + - key: service.name + action: upsert + value: "self-hosted-runner" + - key: github.repository + action: upsert + value: "$GITHUB_REPOSITORY" + - key: github.runner + action: upsert + value: "$RUNNER_NAME" + - key: github.workflow + action: upsert + value: "$GITHUB_WORKFLOW" + - key: github.job + action: upsert + value: "$GITHUB_JOB" + - key: github.run.id + action: upsert + value: "$GITHUB_RUN_ID" + - key: github.run.attempt + action: upsert + value: "$GITHUB_RUN_ATTEMPT" + - key: host.arch + action: upsert + value: "$(uname -m)" transform/self_hosted_runner_loki_labels: log_statements: - context: resource @@ -236,22 +250,20 @@ service: logs/otlp: receivers: [otlp] processors: - - attributes/self_hosted_runner_github_labels - - transform/self_hosted_runner_attributes_to_resource + - resource/self_hosted_runner_github_labels - transform/self_hosted_runner_loki_labels - batch exporters: [otlp/self_hosted_runner] logs/loki: receivers: [loki] processors: - - attributes/self_hosted_runner_github_labels - - transform/self_hosted_runner_attributes_to_resource + - resource/self_hosted_runner_github_labels - transform/self_hosted_runner_loki_labels - batch exporters: [otlp/self_hosted_runner] traces: receivers: [otlp] - processors: [attributes/self_hosted_runner_github_labels, batch] + processors: [resource/self_hosted_runner_github_labels, batch] exporters: [otlp/self_hosted_runner] EOF From d0f4c48a08ec89f4754c2791035fc228d8d38caf Mon Sep 17 00:00:00 2001 From: Weii Wang Date: Wed, 20 May 2026 14:15:49 +0800 Subject: [PATCH 7/7] Update changelog.md --- docs/changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 3e5eac712..2bc6cf0ac 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,11 @@ This changelog documents user-relevant changes to the GitHub runner charm. +## 2026-05-20 + +- Enhanced OpenTelemetry Collector configuration in the pre-job script to expose local OTLP (gRPC/HTTP) and Loki endpoints for workflow to push observability data. +- Enable system.cpu.logical.count and system.cpu.physical.count metrics. + ## 2026-04-24 - Exposed the configured GitHub path (org or repository) as a Terraform module output, allowing consumers to make decisions based on which path a runner is registered to.