diff --git a/e2e/ci_bootstrap_suite.sh b/e2e/ci_bootstrap_suite.sh index 5985db15..95252009 100755 --- a/e2e/ci_bootstrap_suite.sh +++ b/e2e/ci_bootstrap_suite.sh @@ -29,6 +29,8 @@ run_test "bootstrap_sdist_only" run_test "bootstrap_multiple_versions" run_test "bootstrap_multiple_versions_resolve_error" run_test "bootstrap_max_release_age" +run_test "bootstrap_age_constraint_bypass" +run_test "bootstrap_age_fallback_newest" test_section "bootstrap test-mode tests" run_test "mode_resolution" diff --git a/e2e/test_bootstrap_age_constraint_bypass.sh b/e2e/test_bootstrap_age_constraint_bypass.sh new file mode 100755 index 00000000..85c541d0 --- /dev/null +++ b/e2e/test_bootstrap_age_constraint_bypass.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test that constrained (pinned) packages bypass max-release-age filtering. +# A package pinned in constraints should always be built regardless of its age, +# because a constraint pin is explicit user intent that takes precedence over +# the age filter heuristic. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# tomli 2.0.0 was uploaded to PyPI on 2021-12-13. +# We set --max-release-age to a value that EXCLUDES tomli 2.0.0 +# but INCLUDES tomli 2.0.1 (2022-01-02) and 2.0.2 (2025-05-05). +# Then we pin tomli==2.0.0 in constraints — it must still be built. +MAX_AGE=$(python3 -c " +from datetime import date +# Age of tomli 2.0.1 (uploaded 2022-01-02) + 10 day buffer +# This ensures 2.0.1 is inside the window but 2.0.0 is outside +age = (date.today() - date(2022, 1, 2)).days + 10 +print(age) +") + +echo "Using --max-release-age=$MAX_AGE" + +# Create constraints file pinning tomli to the OLD version +constraints_file=$(mktemp) +trap 'rm -f "$constraints_file"; on_exit' EXIT +cat > "$constraints_file" <=2.0,<=2.0.2' + +# Verify that the pinned old version was built despite being outside the age window +echo "" +echo "Checking that constrained (pinned) version was built..." +if find "$OUTDIR/wheels-repo/downloads/" -name "tomli-2.0.0-*.whl" | grep -q .; then + echo "✓ Found wheel for tomli 2.0.0 (constrained — bypassed age filter)" +else + echo "✗ Missing wheel for tomli 2.0.0" + echo "ERROR: tomli 2.0.0 is pinned in constraints and should bypass age filtering" + echo "" + echo "Found wheels:" + find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl' + exit 1 +fi + +# Verify the log confirms the constraint bypass +echo "" +echo "Checking log for constraint bypass..." +if grep -q "skipping age filter for pinned constraint" "$OUTDIR/bootstrap.log"; then + echo "✓ Log confirms age filter was bypassed for pinned constraint" +else + echo "✗ No constraint bypass message found in log" + exit 1 +fi + +echo "" +echo "SUCCESS: Constrained package correctly bypassed age filtering" diff --git a/e2e/test_bootstrap_age_fallback_newest.sh b/e2e/test_bootstrap_age_fallback_newest.sh new file mode 100755 index 00000000..d853d097 --- /dev/null +++ b/e2e/test_bootstrap_age_fallback_newest.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test that multi-version bootstrap with --max-release-age falls back to +# building only the newest version when ALL versions are outside the age window. +# Without this fallback, the bootstrap would fail entirely. + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# Use --max-release-age=1 so ALL tomli versions are outside the window. +# The newest matching version should still be built via the NEWEST fallback. +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + bootstrap \ + --multiple-versions \ + --max-release-age=1 \ + 'tomli>=2.0,<=2.0.2' + +# Count how many tomli wheels were built +TOMLI_COUNT=$(find "$OUTDIR/wheels-repo/downloads/" -name "tomli-*.whl" | wc -l) +echo "Found $TOMLI_COUNT tomli wheel(s)" + +# Exactly one version should be built (the newest fallback) +if [ "$TOMLI_COUNT" -eq 1 ]; then + echo "✓ Exactly one tomli version was built (newest fallback)" +else + echo "✗ Expected exactly 1 tomli version, found $TOMLI_COUNT" + echo "The NEWEST fallback should build only the single newest version" + echo "" + echo "Found wheels:" + find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl' + exit 1 +fi + +# The newest matching version (2.0.2) should be the one built +if find "$OUTDIR/wheels-repo/downloads/" -name "tomli-2.0.2-*.whl" | grep -q .; then + echo "✓ Found wheel for tomli 2.0.2 (newest matching version)" +else + echo "✗ Missing wheel for tomli 2.0.2 — expected the newest version" + echo "" + echo "Found wheels:" + find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl' + exit 1 +fi + +# Verify the log confirms the fallback was triggered +echo "" +echo "Checking log for newest fallback..." +if grep -q "falling back to newest version" "$OUTDIR/bootstrap.log"; then + echo "✓ Log confirms newest version fallback was triggered" +else + echo "✗ No newest-version fallback message found in log" + exit 1 +fi + +echo "" +echo "SUCCESS: Multi-version age fallback correctly built only the newest version" diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 9e2bcfb7..c4943096 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -199,11 +199,16 @@ def _resolve_and_extend( req_type=req_type, ) max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) + age_fallback = ( + resolver.AgeFallback.NEWEST + if self.multiple_versions + else resolver.AgeFallback.ALL + ) results = resolver.find_all_matching_from_provider( provider, req, max_age_cutoff=max_age_cutoff, - fallback_on_empty_age_filter=not self.multiple_versions, + age_fallback=age_fallback, ) if not results and self.multiple_versions and self.cache_wheel_server_url: diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index 574c0131..9f2cfa1b 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -6,6 +6,7 @@ from __future__ import annotations import datetime +import enum import functools import logging import os @@ -60,6 +61,14 @@ ) +class AgeFallback(enum.StrEnum): + """Strategy when max-release-age filtering removes all candidates.""" + + ALL = "all" + NEWEST = "newest" + NONE = "none" + + @functools.lru_cache(maxsize=200) def match_py_req(py_req: str, *, python_version: Version = PYTHON_VERSION) -> bool: """Python version requirement lookup with LRU cache @@ -237,7 +246,7 @@ def find_all_matching_from_provider( provider: BaseProvider, req: Requirement, max_age_cutoff: datetime.datetime | None = None, - fallback_on_empty_age_filter: bool = True, + age_fallback: AgeFallback = AgeFallback.ALL, ) -> list[tuple[str, Version]]: """Find all matching candidates from provider without full dependency resolution. @@ -248,12 +257,15 @@ def find_all_matching_from_provider( provider: The provider to query for candidates. req: The requirement to match. max_age_cutoff: If set, reject candidates published before this time. - If all candidates are older than the cutoff, all are kept and - a warning is emitted to avoid empty resolution. - fallback_on_empty_age_filter: If ``True`` (default), keep all - candidates when age filtering would produce an empty result. - If ``False``, return an empty list instead, allowing the - caller to implement its own fallback strategy. + Age filtering is skipped for packages that have an exact + ``==`` pin in the provider's ``constraints`` object, since + a pin represents explicit user intent that should not be + overridden by a heuristic. Range constraints (``>=``, + ``<``, etc.) still go through age filtering normally. + age_fallback: Strategy when age filtering removes all candidates. + ``ALL`` (default) keeps every candidate with a warning. + ``NEWEST`` keeps only the single newest candidate. + ``NONE`` returns an empty list, letting the caller handle it. Returns list of (url, version) tuples sorted by version (highest first). @@ -291,47 +303,70 @@ def find_all_matching_from_provider( candidates_list = list(candidates) if max_age_cutoff is not None: - logger.info( - "%s: found %d candidate(s) matching %s", - req.name, - len(candidates_list), - req, - ) - max_age_days = (datetime.datetime.now(datetime.UTC) - max_age_cutoff).days - filtered = [ - c - for c in candidates_list - if c.upload_time is None or c.upload_time >= max_age_cutoff - ] - dropped = len(candidates_list) - len(filtered) - if dropped: + # Exact == pins in constraints are explicit user intent — skip age filtering. + constraint = provider.constraints.get_constraint(req.name) + is_pinned = constraint is not None and _has_equality_pin(constraint) + + if is_pinned: logger.info( - "%s: have %d candidate(s) of %s published within %d days", - req.name, - len(filtered), - req, - max_age_days, - ) - if filtered: - candidates_list = filtered - elif fallback_on_empty_age_filter: - logger.warning( - "%s: all %d candidate(s) of %s are older than %d days, " - "keeping all to avoid empty resolution", + "%s: skipping age filter for pinned constraint (%d candidate(s))", req.name, len(candidates_list), - req, - max_age_days, ) else: logger.info( - "%s: all %d candidate(s) of %s are older than %d days", + "%s: found %d candidate(s) matching %s", req.name, len(candidates_list), req, - max_age_days, ) - candidates_list = [] + max_age_days = (datetime.datetime.now(datetime.UTC) - max_age_cutoff).days + filtered = [ + c + for c in candidates_list + if c.upload_time is None or c.upload_time >= max_age_cutoff + ] + dropped = len(candidates_list) - len(filtered) + if dropped: + logger.info( + "%s: have %d candidate(s) of %s published within %d days", + req.name, + len(filtered), + req, + max_age_days, + ) + if filtered: + candidates_list = filtered + elif age_fallback == AgeFallback.ALL: + logger.warning( + "%s: all %d candidate(s) of %s are older than %d days, " + "keeping all to avoid empty resolution", + req.name, + len(candidates_list), + req, + max_age_days, + ) + elif age_fallback == AgeFallback.NEWEST: + newest = candidates_list[0] + logger.info( + "%s: all %d candidate(s) of %s are older than %d days, " + "falling back to newest version %s", + req.name, + len(candidates_list), + req, + max_age_days, + newest.version, + ) + candidates_list = [newest] + else: + logger.info( + "%s: all %d candidate(s) of %s are older than %d days", + req.name, + len(candidates_list), + req, + max_age_days, + ) + candidates_list = [] # Convert candidates to list of (url, version) tuples # Candidates are sorted by version (highest first) by BaseProvider.find_matches() diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index e2a349d8..7222a6fb 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -6,6 +6,7 @@ from packaging.utils import canonicalize_name from packaging.version import Version +from fromager import resolver from fromager.bootstrap_requirement_resolver import BootstrapRequirementResolver from fromager.context import WorkContext from fromager.dependency_graph import DependencyGraph @@ -824,3 +825,69 @@ def test_resolve_skips_cache_fallback_when_no_server_url( ) mock_cache.assert_not_called() + + def test_multi_version_passes_newest_age_fallback( + self, tmp_context: WorkContext + ) -> None: + """Multi-version mode passes AgeFallback.NEWEST to the resolver.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=True, + cache_wheel_server_url="http://cache.test/simple", + ) + req = Requirement("testpkg") + + with ( + patch.object(brr, "_resolve_from_graph", return_value=None), + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider", + ), + patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[("url", Version("1.0"))], + ) as mock_find, + ): + brr.resolve( + req, + RequirementType.INSTALL, + parent_req=None, + pre_built=False, + return_all_versions=True, + ) + + mock_find.assert_called_once() + call_kwargs = mock_find.call_args.kwargs + assert call_kwargs["age_fallback"] == resolver.AgeFallback.NEWEST + + def test_single_version_passes_all_age_fallback( + self, tmp_context: WorkContext + ) -> None: + """Single-version mode passes AgeFallback.ALL to the resolver.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=False, + ) + req = Requirement("testpkg") + + with ( + patch.object(brr, "_resolve_from_graph", return_value=None), + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider", + ), + patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[("url", Version("1.0"))], + ) as mock_find, + ): + brr.resolve( + req, + RequirementType.INSTALL, + parent_req=None, + pre_built=False, + ) + + mock_find.assert_called_once() + call_kwargs = mock_find.call_args.kwargs + assert call_kwargs["age_fallback"] == resolver.AgeFallback.ALL diff --git a/tests/test_cooldown.py b/tests/test_cooldown.py index 0d47cc5f..151f9eaf 100644 --- a/tests/test_cooldown.py +++ b/tests/test_cooldown.py @@ -18,7 +18,15 @@ from packaging.requirements import Requirement from packaging.version import Version -from fromager import candidate, context, packagesettings, resolver, sources, wheels +from fromager import ( + candidate, + constraints, + context, + packagesettings, + resolver, + sources, + wheels, +) from fromager.requirements_file import RequirementType _BOOTSTRAP_TIME = datetime.datetime(2026, 3, 26, 0, 0, 0, tzinfo=datetime.UTC) @@ -796,10 +804,10 @@ def test_max_release_age_all_too_old_keeps_all( assert "keeping all to avoid empty resolution" in caplog.text -def test_max_release_age_all_too_old_returns_empty_when_fallback_disabled( +def test_max_release_age_all_too_old_returns_empty_when_fallback_none( caplog: pytest.LogCaptureFixture, ) -> None: - """When fallback is disabled and all versions are too old, return empty list.""" + """With AgeFallback.NONE and all versions too old, return empty list.""" max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) with requests_mock.Mocker() as r: r.get( @@ -813,13 +821,38 @@ def test_max_release_age_all_too_old_returns_empty_when_fallback_disabled( provider, Requirement("test-pkg"), max_age_cutoff=max_age_cutoff, - fallback_on_empty_age_filter=False, + age_fallback=resolver.AgeFallback.NONE, ) assert results == [] assert "all 3 candidate(s)" in caplog.text assert "keeping all to avoid empty resolution" not in caplog.text +def test_max_release_age_falls_back_to_newest( + caplog: pytest.LogCaptureFixture, +) -> None: + """With AgeFallback.NEWEST and all versions too old, return only the newest.""" + max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, + Requirement("test-pkg"), + max_age_cutoff=max_age_cutoff, + age_fallback=resolver.AgeFallback.NEWEST, + ) + versions = [str(v) for _, v in results] + assert versions == ["2.0.0"] + assert "falling back to newest version 2.0.0" in caplog.text + assert "keeping all to avoid empty resolution" not in caplog.text + + def test_max_release_age_candidates_without_upload_time_pass_through() -> None: """Candidates without upload_time are not filtered out by max-release-age.""" no_timestamp_response = { @@ -965,3 +998,117 @@ def test_resolve_package_cooldown_toplevel_compound_specifier_not_exempt( ctx, Requirement("test-pkg==1.0,>0.9"), req_type=RequirementType.TOP_LEVEL ) assert result is _COOLDOWN + + +# --------------------------------------------------------------------------- +# constraint pin bypass tests — exact == pins skip age filtering +# --------------------------------------------------------------------------- + + +def test_max_release_age_skips_filter_for_pinned_constraint( + caplog: pytest.LogCaptureFixture, +) -> None: + """Exact == pin in constraints bypasses age filtering.""" + # Cutoff in the future so ALL versions are "too old" + max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) + c = constraints.Constraints() + c.add_constraint("test-pkg==1.2.2") + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True, constraints=c) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, + Requirement("test-pkg"), + max_age_cutoff=max_age_cutoff, + age_fallback=resolver.AgeFallback.NONE, + ) + + # The pin restricts candidates to 1.2.2, and age filtering is + # bypassed — so 1.2.2 must survive even though it's outside the window. + versions = [str(v) for _, v in results] + assert "1.2.2" in versions + assert "skipping age filter for pinned constraint" in caplog.text + assert "keeping all to avoid empty resolution" not in caplog.text + + +def test_max_release_age_skips_filter_for_pinned_constraint_with_newest_fallback( + caplog: pytest.LogCaptureFixture, +) -> None: + """Exact == pin bypasses age filtering even with NEWEST fallback.""" + max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) + c = constraints.Constraints() + c.add_constraint("test-pkg==1.2.2") + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True, constraints=c) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, + Requirement("test-pkg"), + max_age_cutoff=max_age_cutoff, + age_fallback=resolver.AgeFallback.NEWEST, + ) + + versions = [str(v) for _, v in results] + assert "1.2.2" in versions + assert "skipping age filter for pinned constraint" in caplog.text + assert "falling back to newest version" not in caplog.text + + +def test_max_release_age_still_filters_range_constrained_package( + caplog: pytest.LogCaptureFixture, +) -> None: + """Range constraints (not exact pins) are still subject to age filtering.""" + max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) + c = constraints.Constraints() + c.add_constraint("test-pkg>=1.0") + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True, constraints=c) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, + Requirement("test-pkg"), + max_age_cutoff=max_age_cutoff, + age_fallback=resolver.AgeFallback.NONE, + ) + + assert results == [] + assert "skipping age filter for pinned constraint" not in caplog.text + + +def test_max_release_age_still_filters_unconstrained_package( + caplog: pytest.LogCaptureFixture, +) -> None: + """Unconstrained packages are still subject to age filtering.""" + max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, + Requirement("test-pkg"), + max_age_cutoff=max_age_cutoff, + age_fallback=resolver.AgeFallback.NONE, + ) + + assert results == [] + assert "skipping age filter for pinned constraint" not in caplog.text