feat: provider label discovery and catalog test_id metadata

isvctl/src/isvctl/cli/catalog.py

@@ -20,10 +20,11 @@
 
 import json
 import logging
+from collections import Counter
 from typing import Annotated
 
 import typer
-from isvtest.catalog import build_catalog, get_catalog_version
+from isvtest.catalog import build_catalog, build_label_file_map, get_catalog_version
 from isvtest.release_manifest import load_released_tests
 from rich.console import Console
 from rich.table import Table
@@ -84,21 +85,87 @@ def list_cmd(
         padding=(0, 1),
     )
     table.add_column("Test", style="green", no_wrap=True)
-    table.add_column("Platforms", style="cyan")
-    table.add_column("Labels", style="dim")
+    table.add_column("Test IDs", style="magenta", max_width=32)
+    table.add_column("Labels (Platforms)", style="dim", max_width=40)
     table.add_column("Description")
 
     for entry in sorted(catalog_entries, key=lambda e: e["name"]):
+        labels = ", ".join(entry.get("labels") or [])
+        platforms = ", ".join(entry.get("platforms") or [])
+        if labels and platforms:
+            labels_platforms = f"{labels} ({platforms})"
+        else:
+            labels_platforms = labels or platforms
         table.add_row(
             entry["name"],
-            ", ".join(entry.get("platforms") or []) or "-",
-            ", ".join(entry.get("labels") or []) or "-",
+            ", ".join(entry.get("test_ids") or []) or "-",
+            labels_platforms or "-",
             entry.get("description") or "-",
         )
 
     console.print(table)
 
 
+@app.command("labels")
+def labels_cmd(
+    json_output: Annotated[
+        bool,
+        typer.Option("--json", help="Emit the labels as JSON instead of a table"),
+    ] = False,
+    show_files: Annotated[
+        bool,
+        typer.Option("--files", help="Also show the config file(s) declaring each label"),
+    ] = False,
+) -> None:
+    """List every label in the catalog with the number of tests carrying it.
+
+    Released tests only by default. Set ``ISVTEST_INCLUDE_UNRELEASED=1`` to
+    include unreleased validations (matches the gate used at run time). Pass
+    ``--files`` to also list the config file(s) that declare each label.
+
+    Examples:
+        isvctl catalog labels
+        isvctl catalog labels --files
+        isvctl catalog labels --json
+        ISVTEST_INCLUDE_UNRELEASED=1 isvctl catalog labels
+    """
+    counts = Counter(label for entry in build_catalog() for label in (entry.get("labels") or []))
+    sorted_counts = sorted(counts.items())
+    label_files = build_label_file_map() if show_files else {}
+
+    def files_for(label: str) -> list[str]:
+        """Return the sorted config files declaring ``label`` (empty without --files)."""
+        return sorted(label_files.get(label, set()))
+
+    if json_output:
+        labels = [
+            {"label": label, "tests": count, **({"files": files_for(label)} if show_files else {})}
+            for label, count in sorted_counts
+        ]
+        typer.echo(json.dumps({"labels": labels}, indent=2))
+        return
+
+    table = Table(
+        title=f"Catalog Labels ({len(counts)} labels)",
+        title_justify="left",
+        show_header=True,
+        header_style="bold",
+        padding=(0, 1),
+    )
+    table.add_column("Label", style="green", no_wrap=True)
+    table.add_column("Tests", style="cyan", justify="right")
+    if show_files:
+        table.add_column("Files", style="dim")
+
+    for label, count in sorted_counts:
+        row = [label, str(count)]
+        if show_files:
+            row.append("\n".join(files_for(label)) or "-")
+        table.add_row(*row)
+
+    console.print(table)
+
+
 @app.command("push")
 def push(
     verbose: Annotated[

isvctl/src/isvctl/cli/test.py

@@ -23,11 +23,12 @@
 import sys
 from datetime import UTC, datetime
 from pathlib import Path
-from typing import Annotated, TextIO
+from typing import Annotated, Any, TextIO
 
 import typer
 import yaml
 from isvtest.catalog import build_catalog, get_catalog_version
+from isvtest.release_manifest import load_released_test_filter
 
 from isvctl.cli import setup_logging
 from isvctl.cli.common import (
@@ -38,13 +39,20 @@
     print_progress,
     print_warning,
 )
+from isvctl.config.label_discovery import (
+    ProviderConfigMatch,
+    available_labels,
+    discover_provider_label_configs,
+    list_providers,
+)
 from isvctl.config.merger import merge_yaml_files
 from isvctl.config.schema import RunConfig
 from isvctl.orchestrator.loop import Orchestrator, Phase
 from isvctl.redaction import redact_dict
 from isvctl.reporting import check_upload_credentials, create_test_run, get_environment_config, update_test_run
 
 logger = logging.getLogger(__name__)
+CONFIGS_ROOT = Path(__file__).resolve().parents[3] / "configs"
 
 
 class TeeWriter:
@@ -78,11 +86,40 @@ def isatty(self) -> bool:
 )
 
 
+def _provider_discovery_plan(provider: str, labels: list[str], matches: list[ProviderConfigMatch]) -> dict[str, Any]:
+    """Return a JSON-serializable provider label discovery plan."""
+    return {
+        "provider": provider,
+        "labels": labels,
+        "configs": [
+            {
+                "config": str(match.config_path),
+                "matched_checks": [
+                    {
+                        "category": check.category,
+                        "name": check.name,
+                        "labels": list(check.labels),
+                    }
+                    for check in match.matched_checks
+                ],
+            }
+            for match in matches
+        ],
+    }
+
+
+def _junitxml_for_discovered_config(junitxml: Path, match: ProviderConfigMatch, total: int) -> Path:
+    """Return a non-overlapping JUnit path for a discovered config run."""
+    if total <= 1:
+        return junitxml
+    return junitxml.with_name(f"{junitxml.stem}-{match.config_path.stem}{junitxml.suffix}")
+
+
 @app.command("run", context_settings={"allow_extra_args": True, "ignore_unknown_options": True})
 def run(
     ctx: typer.Context,
     config_files: Annotated[
-        list[Path],
+        list[Path] | None,
         typer.Option(
             "--config",
             "-f",
@@ -92,7 +129,14 @@ def run(
             dir_okay=False,
             readable=True,
         ),
-    ],
+    ] = None,
+    provider: Annotated[
+        str | None,
+        typer.Option(
+            "--provider",
+            help="Provider name for label discovery when no --config/-f files are supplied.",
+        ),
+    ] = None,
     set_values: Annotated[
         list[str] | None,
         typer.Option(
@@ -208,9 +252,65 @@ def run(
     setup_logging(verbose)
     apply_user_config(no_user_config)
 
+    if provider:
+        if config_files:
+            print_error("--provider discovery cannot be combined with --config/-f.")
+            raise typer.Exit(code=1)
+        if not labels:
+            print_error("--provider requires at least one --label/-l for discovery.")
+            raise typer.Exit(code=1)
+
+        known_providers = list_providers(CONFIGS_ROOT)
+        if provider not in known_providers:
+            print_error(f"Unknown provider {provider!r}. Available providers: {', '.join(known_providers)}")
+            raise typer.Exit(code=1)
+
+        matches = discover_provider_label_configs(
+            provider, labels, configs_root=CONFIGS_ROOT, released_tests=load_released_test_filter()
+        )
+        if not matches:
+            known_labels = available_labels(provider, configs_root=CONFIGS_ROOT)
+            print_error(
+                f"No {provider!r} provider configs match labels: {', '.join(labels)}. "
+                f"Available labels for {provider!r}: {', '.join(sorted(known_labels))}"
+            )
+            raise typer.Exit(code=1)
+
+        if dry_run:
+            typer.echo(json.dumps(_provider_discovery_plan(provider, labels, matches), indent=2))
+            return
+
+        print_progress(
+            f"Discovered {len(matches)} {provider!r} provider config(s) matching labels: {', '.join(labels)}"
+        )
+        for match in matches:
+            print_progress(f"\n--- Running {match.config_path} ---")
+            run(
+                ctx,
+                config_files=[match.config_path],
+                provider=None,
+                set_values=set_values,
+                phase=phase,
+                labels=labels,
+                dry_run=False,
+                working_dir=working_dir,
+                verbose=verbose,
+                no_user_config=no_user_config,
+                junitxml=_junitxml_for_discovered_config(junitxml, match, len(matches)),
+                color=color,
+                no_upload=no_upload,
+                lab_id=lab_id,
+                tags=tags,
+                isv_software_version=isv_software_version,
+            )
+        return
+
     # Validate at least one config file is provided
     if not config_files:
-        print_error("At least one --config/-f config file is required.")
+        if labels:
+            print_error("--label requires either --provider (for label discovery) or --config/-f.")
+        else:
+            print_error("At least one --config/-f config file is required.")
         raise typer.Exit(code=1)
 
     # Collect extra pytest args from context (after --)
@@ -353,8 +453,11 @@ def run(
     # Update test run after tests complete
     if upload_results and test_run_id and lab_id:
         print_progress("Uploading test results to ISV Lab Service...")
-        # Look for junit XML in _output, working directory, or current directory
-        junit_path = output_dir / "junit-validation.xml"
+        # Prefer the requested --junitxml (provider discovery gives each config
+        # its own report name), then fall back to _output, working dir, or cwd.
+        junit_path = junitxml
+        if not junit_path.exists():
+            junit_path = output_dir / "junit-validation.xml"
         if not junit_path.exists():
             junit_path = effective_working_dir / "junit-validation.xml"
         if not junit_path.exists():

isvctl/src/isvctl/config/label_discovery.py

@@ -0,0 +1,91 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Provider-scoped label discovery helpers."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from isvtest.core.resolution import ValidationEntry, parse_validations, resolve_class_key
+
+from isvctl.config.merger import merge_yaml_files
+
+
+def _iter_config_validations(config_path: Path) -> Iterator[ValidationEntry]:
+    """Yield the validation entries of a config with its imports resolved."""
+    merged = merge_yaml_files([config_path])
+    raw_validations = (merged.get("tests") or {}).get("validations") or {}
+    yield from parse_validations(raw_validations)
+
+
+@dataclass(frozen=True)
+class MatchedCheck:
+    """A validation check that matched requested labels."""
+
+    category: str
+    name: str
+    labels: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class ProviderConfigMatch:
+    """A provider config selected by label discovery."""
+
+    config_path: Path
+    matched_checks: tuple[MatchedCheck, ...]
+
+
+def list_providers(configs_root: Path) -> list[str]:
+    """Return provider names that expose a discoverable ``config/*.yaml`` directory."""
+    providers_dir = configs_root / "providers"
+    if not providers_dir.is_dir():
+        return []
+    return sorted(
+        provider_dir.name
+        for provider_dir in providers_dir.iterdir()
+        if provider_dir.is_dir() and any((provider_dir / "config").glob("*.yaml"))
+    )
+
+
+def available_labels(provider: str, *, configs_root: Path) -> set[str]:
+    """Return every label declared across a provider's resolved config wiring."""
+    provider_config_dir = configs_root / "providers" / provider / "config"
+    labels: set[str] = set()
+    for config_path in provider_config_dir.glob("*.yaml"):
+        for entry in _iter_config_validations(config_path):
+            labels.update(entry.labels)
+    return labels
+
+
+def discover_provider_label_configs(
+    provider: str,
+    labels: list[str],
+    *,
+    configs_root: Path,
+    released_tests: set[str] | None = None,
+) -> list[ProviderConfigMatch]:
+    """Return provider configs whose resolved validation wiring matches all labels.
+
+    A check counts toward a match only if it is also runnable under the release
+    filter, mirroring orchestrator execution: when ``released_tests`` is a set,
+    unreleased checks are ignored so a config is not selected solely on a check
+    that would be skipped at runtime. ``None`` disables the filter (include all),
+    matching ``ISVTEST_INCLUDE_UNRELEASED``.
+    """
+    requested = {label for label in labels if label}
+    provider_config_dir = configs_root / "providers" / provider / "config"
+    matches: list[ProviderConfigMatch] = []
+
+    for config_path in sorted(provider_config_dir.glob("*.yaml")):
+        matched_checks = tuple(
+            MatchedCheck(category=entry.category, name=entry.name, labels=entry.labels)
+            for entry in _iter_config_validations(config_path)
+            if requested.issubset(entry.labels)
+            and (released_tests is None or resolve_class_key(entry.name, released_tests) is not None)
+        )
+        if matched_checks:
+            matches.append(ProviderConfigMatch(config_path=config_path, matched_checks=matched_checks))
+    return matches