From 670cfb6853989486489268ed5fb9ddaa2064882f Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 3 Jul 2026 16:32:55 +0200 Subject: [PATCH] Add prek hook to keep the Go toolchain version in sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go toolchain version is pinned in eight places (go-sdk/go.mod, the go_example module, the setup-go step in both CI workflow copies, the golang alpine builder image used by the e2e conftest and by breeze, and the default_language_version.golang in the top-level and go-sdk prek configs). They have no cross-file include, so a dependency bump that raises the go directive in go-sdk/go.mod leaves the rest behind and the drift only surfaces as an opaque "go.mod requires go >= " failure deep in CI — as happened in PR #69214. Add a prek hook that treats go-sdk/go.mod as the single source of truth and fails when any other site disagrees at major.minor granularity, pointing directly at the drifting file. --- .pre-commit-config.yaml | 15 ++ scripts/ci/prek/check_go_version_in_sync.py | 197 ++++++++++++++++++ .../ci/prek/test_check_go_version_in_sync.py | 130 ++++++++++++ 3 files changed, 342 insertions(+) create mode 100755 scripts/ci/prek/check_go_version_in_sync.py create mode 100644 scripts/tests/ci/prek/test_check_go_version_in_sync.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 42d9ffcbe1a66..662a4d2e80a61 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -297,6 +297,21 @@ repos: ^Dockerfile\.ci$ pass_filenames: false require_serial: true + - id: check-go-version-in-sync + name: Check Go toolchain version is consistent across build files + entry: ./scripts/ci/prek/check_go_version_in_sync.py + language: python + files: > + (?x) + ^go-sdk/go\.mod$| + ^kubernetes-tests/lang_sdk/go_example/go\.mod$| + ^\.github/workflows/ci-(arm|amd)\.yml$| + ^airflow-e2e-tests/tests/airflow_e2e_tests/constants\.py$| + ^dev/breeze/src/airflow_breeze/commands/kubernetes_commands\.py$| + ^go-sdk/\.pre-commit-config\.yaml$| + ^\.pre-commit-config\.yaml$ + pass_filenames: false + require_serial: true - id: check-partition-mapper-defaults-in-sync name: Check partition-mapper core/SDK sync (FanOutMapper table + SegmentWindow/FixedKeyMapper) entry: ./scripts/ci/prek/check_partition_mapper_defaults_in_sync.py diff --git a/scripts/ci/prek/check_go_version_in_sync.py b/scripts/ci/prek/check_go_version_in_sync.py new file mode 100755 index 0000000000000..250bb2246b251 --- /dev/null +++ b/scripts/ci/prek/check_go_version_in_sync.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Keep the Go toolchain version in sync across every file that pins it. + +``golang.org/x/net`` (and other ``golang.org/x`` modules) periodically raise +their minimum Go version. When that happens a dependency bump changes the +``go`` directive in ``go-sdk/go.mod`` but leaves the *other* places that pin +the toolchain untouched — and CI then fails deep inside static checks and the +Go SDK e2e bundle build with ``go.mod requires go >= `` (this is exactly +what happened in PR #69214). Those pins have no cross-file include, so nothing +catches the drift until CI does. + +The single source of truth is the ``go`` directive in ``go-sdk/go.mod``. Every +other site below must agree with it at ``major.minor`` granularity (the minor +version is what selects the toolchain and the builder image; the go.mod patch +suffix is not meaningful for those): + +- ``go-sdk/go.mod`` -> ``go `` (SOURCE OF TRUTH) +- ``kubernetes-tests/lang_sdk/go_example/go.mod`` -> ``go `` (resolves the SDK via ``replace``) +- ``.github/workflows/ci-amd.yml`` -> Setup Go ``go-version: `` +- ``.github/workflows/ci-arm.yml`` -> Setup Go ``go-version: `` +- ``airflow-e2e-tests/tests/airflow_e2e_tests/constants.py`` -> ``GO_BUILDER_IMAGE`` default ``golang:-alpine`` +- ``dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py`` -> ``LANG_SDK_GO_BUILDER_IMAGE`` default ``golang:-alpine`` +- ``go-sdk/.pre-commit-config.yaml`` -> ``default_language_version.golang`` (the toolchain prek uses for the SDK's ``language: golang`` hooks — this is the pin that broke static checks in #69214, not the setup-go one) +- ``.pre-commit-config.yaml`` -> ``default_language_version.golang`` (top-level default for any ``language: golang`` hook) + +When they disagree, bump the drifting sites to the source-of-truth minor +version. If you add a new place that pins the Go version, register it here too. + +Run from the repo root: + + uv run --project scripts python scripts/ci/prek/check_go_version_in_sync.py + +Exits 0 if every site agrees with the source of truth, 1 otherwise. +""" + +from __future__ import annotations + +import dataclasses +import pathlib +import re +import sys + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + +# The ``go`` directive: ``go 1.25`` or ``go 1.25.0``. +GO_MOD_DIRECTIVE = re.compile(r"^go\s+(\d+\.\d+(?:\.\d+)?)\s*$", re.MULTILINE) +# Setup Go step: ``go-version: 1.25`` (optionally quoted). +WORKFLOW_GO_VERSION = re.compile(r"^\s*go-version:\s*[\"']?(\d+\.\d+(?:\.\d+)?)[\"']?\s*$", re.MULTILINE) +# Builder image default: ``golang:1.25-alpine``. +GOLANG_BUILDER_IMAGE = re.compile(r"golang:(\d+\.\d+(?:\.\d+)?)-alpine") +# prek ``default_language_version`` entry: `` golang: 1.25.0``. +PREK_GOLANG_VERSION = re.compile(r"^\s*golang:\s*(\d+\.\d+(?:\.\d+)?)\s*$", re.MULTILINE) + + +@dataclasses.dataclass +class VersionSite: + """A single location that pins the Go toolchain version.""" + + label: str + path: pathlib.Path + pattern: re.Pattern[str] + is_source_of_truth: bool = False + + def extract(self) -> str | None: + """Return the captured version string, or ``None`` if the pattern is missing.""" + if not self.path.exists(): + return None + if m := self.pattern.search(self.path.read_text()): + return m.group(1) + return None + + +def major_minor(version: str) -> str: + """Reduce ``1.25`` / ``1.25.0`` to its ``major.minor`` (``1.25``).""" + parts = version.split(".") + return ".".join(parts[:2]) + + +def build_sites(repo_root: pathlib.Path) -> list[VersionSite]: + """Build the list of Go-version pin sites, resolved against ``repo_root``.""" + workflows = repo_root / ".github" / "workflows" + return [ + VersionSite( + label="go-sdk/go.mod (go directive)", + path=repo_root / "go-sdk" / "go.mod", + pattern=GO_MOD_DIRECTIVE, + is_source_of_truth=True, + ), + VersionSite( + label="kubernetes-tests/lang_sdk/go_example/go.mod (go directive)", + path=repo_root / "kubernetes-tests" / "lang_sdk" / "go_example" / "go.mod", + pattern=GO_MOD_DIRECTIVE, + ), + VersionSite( + label=".github/workflows/ci-amd.yml (Setup Go go-version)", + path=workflows / "ci-amd.yml", + pattern=WORKFLOW_GO_VERSION, + ), + VersionSite( + label=".github/workflows/ci-arm.yml (Setup Go go-version)", + path=workflows / "ci-arm.yml", + pattern=WORKFLOW_GO_VERSION, + ), + VersionSite( + label="airflow-e2e-tests/.../constants.py (GO_BUILDER_IMAGE)", + path=repo_root / "airflow-e2e-tests" / "tests" / "airflow_e2e_tests" / "constants.py", + pattern=GOLANG_BUILDER_IMAGE, + ), + VersionSite( + label="dev/breeze/.../kubernetes_commands.py (LANG_SDK_GO_BUILDER_IMAGE)", + path=repo_root + / "dev" + / "breeze" + / "src" + / "airflow_breeze" + / "commands" + / "kubernetes_commands.py", + pattern=GOLANG_BUILDER_IMAGE, + ), + VersionSite( + label="go-sdk/.pre-commit-config.yaml (default_language_version.golang)", + path=repo_root / "go-sdk" / ".pre-commit-config.yaml", + pattern=PREK_GOLANG_VERSION, + ), + VersionSite( + label=".pre-commit-config.yaml (default_language_version.golang)", + path=repo_root / ".pre-commit-config.yaml", + pattern=PREK_GOLANG_VERSION, + ), + ] + + +def check_sync(sites: list[VersionSite], repo_root: pathlib.Path) -> tuple[int, str]: + """Compare every site against the source of truth. Returns ``(exit_code, report)``.""" + results = [(site, site.extract()) for site in sites] + + if missing := [site for site, version in results if version is None]: + lines = [f"ERROR: Go version pin not found in {site.path.relative_to(repo_root)}" for site in missing] + lines += [f" (expected pattern {site.pattern.pattern!r})" for site in missing] + return 1, "\n".join(lines) + + source = next(site for site, _ in results if site.is_source_of_truth) + expected = major_minor(source.extract()) # type: ignore[arg-type] + + drifted = [(site, version) for site, version in results if major_minor(version) != expected] # type: ignore[arg-type] + if not drifted: + return 0, f"OK: Go version is consistently {expected} across all {len(sites)} pin sites." + + col = max(len(site.label) for site, _ in results) + lines = [ + f"ERROR: Go version drifted from the source of truth ({source.label} = {expected}).", + "", + f" {'PIN SITE':<{col}} VERSION", + ] + for site, version in results: + marker = ( + " <- source of truth" + if site.is_source_of_truth + else ( + " <- DRIFT" if major_minor(version) != expected else "" # type: ignore[arg-type] + ) + ) + lines.append(f" {site.label:<{col}} {version}{marker}") + lines += [ + "", + f"Bump the drifting site(s) to {expected} to match {source.label},", + "or, if you intentionally changed the source of truth, update the other pins to match.", + "After changing a go.mod, run its `go mod tidy` so go.sum stays consistent.", + ] + return 1, "\n".join(lines) + + +def main() -> int: + exit_code, report = check_sync(build_sites(REPO_ROOT), REPO_ROOT) + print(report) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/prek/test_check_go_version_in_sync.py b/scripts/tests/ci/prek/test_check_go_version_in_sync.py new file mode 100644 index 0000000000000..8092f17e034a4 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_go_version_in_sync.py @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from pathlib import Path + +import pytest +from check_go_version_in_sync import build_sites, check_sync, major_minor + +# Default (all-consistent) version at every pin site. Individual tests override +# one entry to simulate drift. go.mod carries the patch suffix; the workflow and +# image sites carry only major.minor — the check must treat these as equal. +DEFAULT_VERSIONS = { + "go_sdk_mod": "1.25.0", + "go_example_mod": "1.25.0", + "ci_amd": "1.25", + "ci_arm": "1.25", + "constants": "1.25", + "kubernetes_commands": "1.25", + "go_sdk_prek": "1.25.0", + "root_prek": "1.25.0", +} + + +def _write_tree(root: Path, versions: dict[str, str]) -> None: + """Materialise a minimal repo tree with a Go version pin at each of the eight sites.""" + + def write(rel: str, content: str) -> None: + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + + write("go-sdk/go.mod", f"module github.com/apache/airflow/go-sdk\n\ngo {versions['go_sdk_mod']}\n") + write( + "kubernetes-tests/lang_sdk/go_example/go.mod", + f"module github.com/apache/airflow/kubernetes-tests/lang_sdk/go_example\n\n" + f"go {versions['go_example_mod']}\n", + ) + for site, rel in (("ci_amd", "ci-amd.yml"), ("ci_arm", "ci-arm.yml")): + write( + f".github/workflows/{rel}", + f" - name: Setup Go\n with:\n go-version: {versions[site]}\n", + ) + write( + "airflow-e2e-tests/tests/airflow_e2e_tests/constants.py", + f'GO_BUILDER_IMAGE = os.environ.get("GO_BUILDER_IMAGE", "golang:{versions["constants"]}-alpine")\n', + ) + write( + "dev/breeze/src/airflow_breeze/commands/kubernetes_commands.py", + f'LANG_SDK_GO_BUILDER_IMAGE = os.environ.get("GO_BUILDER_IMAGE", ' + f'"golang:{versions["kubernetes_commands"]}-alpine")\n', + ) + for site, rel in ( + ("go_sdk_prek", "go-sdk/.pre-commit-config.yaml"), + ("root_prek", ".pre-commit-config.yaml"), + ): + write( + rel, + f"default_language_version:\n python: python3\n node: 22.19.0\n golang: {versions[site]}\n", + ) + + +@pytest.mark.parametrize( + ("version", "expected"), + [("1.25", "1.25"), ("1.25.0", "1.25"), ("1.24.6", "1.24")], +) +def test_major_minor(version: str, expected: str): + assert major_minor(version) == expected + + +def test_all_sites_consistent(tmp_path: Path): + _write_tree(tmp_path, DEFAULT_VERSIONS) + exit_code, report = check_sync(build_sites(tmp_path), tmp_path) + assert exit_code == 0 + assert "consistently 1.25" in report + + +def test_go_mod_patch_suffix_matches_major_minor_pins(tmp_path: Path): + """go.mod may pin 1.25.0 while the workflow/image sites pin 1.25 — that is in sync.""" + _write_tree(tmp_path, {**DEFAULT_VERSIONS, "go_sdk_mod": "1.25.0", "ci_amd": "1.25"}) + exit_code, _ = check_sync(build_sites(tmp_path), tmp_path) + assert exit_code == 0 + + +@pytest.mark.parametrize( + "drifted_site", + ["go_example_mod", "ci_amd", "ci_arm", "constants", "kubernetes_commands", "go_sdk_prek", "root_prek"], +) +def test_drift_in_any_derived_site_is_flagged(tmp_path: Path, drifted_site: str): + """A single derived site left on the old minor version must fail the check.""" + _write_tree( + tmp_path, {**DEFAULT_VERSIONS, drifted_site: "1.24" if "mod" not in drifted_site else "1.24.6"} + ) + exit_code, report = check_sync(build_sites(tmp_path), tmp_path) + assert exit_code == 1 + assert "drifted from the source of truth" in report + assert "<- DRIFT" in report + + +def test_source_of_truth_bump_flags_all_stale_sites(tmp_path: Path): + """Bumping only go-sdk/go.mod (the source of truth) flags every other site — the #69214 scenario.""" + _write_tree(tmp_path, {**DEFAULT_VERSIONS, "go_sdk_mod": "1.26.0"}) + exit_code, report = check_sync(build_sites(tmp_path), tmp_path) + assert exit_code == 1 + # All seven non-source sites are still on the old version -> all flagged. + assert report.count("<- DRIFT") == 7 + assert "<- source of truth" in report + + +def test_missing_pin_is_reported(tmp_path: Path): + _write_tree(tmp_path, DEFAULT_VERSIONS) + (tmp_path / "go-sdk" / "go.mod").write_text("module github.com/apache/airflow/go-sdk\n") + exit_code, report = check_sync(build_sites(tmp_path), tmp_path) + assert exit_code == 1 + assert "Go version pin not found" in report + assert "go-sdk/go.mod" in report