From 21f6eb2d442df4c87c3b0ae405c6bdcaa57cd301 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Tue, 30 Jun 2026 22:06:38 +0200 Subject: [PATCH 01/28] Experiment workflow works --- src/data_input/__init__.py | 10 ++++++++++ src/verification/__init__.py | 21 +++++++++++++++------ src/verification/spatial.py | 5 +++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index d9d101ab..ae343ca2 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -126,6 +126,16 @@ def load_analysis_data_from_zarr( if "cell" in ds.dims: ds = ds.rename({"cell": "values"}) + # Drop grid points with undefined (NaN) coordinates + if "values" in ds.dims and "latitude" in ds.coords and "longitude" in ds.coords: + valid = np.isfinite(ds["latitude"].values) & np.isfinite(ds["longitude"].values) + if not valid.all(): + LOG.warning( + "Dropping %d grid point(s) with undefined lat/lon from truth dataset.", + int((~valid).sum()), + ) + ds = ds.isel(values=valid) + times = np.datetime64(reftime) + np.asarray(steps, dtype="timedelta64[h]") return _select_valid_times(ds, times) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index a4ee8499..7f3356da 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -45,11 +45,12 @@ def __init__( ).transform regions = {} - # add inner region for ML evaluation - regions["all"] = [ - Polygon(list(zip([1.5, 16, 16, 1.5, 1.5], [43, 43, 49.5, 49.5, 43]))) - ] - if shp and shp != [""]: + has_shapefiles = bool(shp and shp != [""]) + if has_shapefiles: + # With explicit regional shapefiles, restrict "all" to the Alpine inner domain + regions["all"] = [ + Polygon(list(zip([1.5, 16, 16, 1.5, 1.5], [43, 43, 49.5, 49.5, 43]))) + ] shp = [shp] if isinstance(shp, str) else shp for shapefile in shp: region_name = Path(shapefile).stem @@ -57,12 +58,20 @@ def __init__( regions[region_name] = [ transform(proj, record.geometry) for record in reader.records() ] + else: + # No shapefile regions: "all" covers the full domain (e.g. global evaluation) + regions["all"] = None self.regions = regions def get_masks(self, lat: xr.DataArray, lon: xr.DataArray) -> xr.DataArray: masks = [] for region_name, polygons in self.regions.items(): - mask = self._mask_from_polygons(polygons, lat, lon) + if polygons is None: + mask = xr.DataArray( + np.ones(lon.shape, dtype=bool), coords=lon.coords, dims=lon.dims + ) + else: + mask = self._mask_from_polygons(polygons, lat, lon) masks.append(mask.assign_coords(region=region_name)) return xr.concat(masks, dim="region") diff --git a/src/verification/spatial.py b/src/verification/spatial.py index c589ee75..f5babc20 100644 --- a/src/verification/spatial.py +++ b/src/verification/spatial.py @@ -141,7 +141,7 @@ def map_forecast_to_truth(fcst: xr.Dataset, truth: xr.Dataset) -> xr.Dataset: "latitude": (fcst["latitude"].dims, truth["latitude"].data), "longitude": (fcst["longitude"].dims, truth["longitude"].data), } - if "values" in fcst.dims and "values" in truth.dims: + if "values" in fcst.dims and "values" in truth.coords: coords["values"] = truth["values"].data return fcst.assign_coords(coords) @@ -163,7 +163,8 @@ def map_forecast_to_truth(fcst: xr.Dataset, truth: xr.Dataset) -> xr.Dataset: fcst = fcst.drop_vars(["x", "y", "values"], errors="ignore") fcst = fcst.assign_coords(longitude=("values", truth["longitude"].data)) fcst = fcst.assign_coords(latitude=("values", truth["latitude"].data)) - fcst = fcst.assign_coords(values=truth["values"]) + if "values" in truth.coords: + fcst = fcst.assign_coords(values=truth["values"]) if truth_is_grid: fcst = fcst.unstack("values") From d8acf3c78e29a47fa3fd68c4a7d3dac3e268f7b8 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Tue, 30 Jun 2026 22:09:20 +0200 Subject: [PATCH 02/28] Add example config --- config/stage-a-o96-multi-step.yaml | 50 +++++++++++++++++++ .../o96-global-multistep-forecaster.yaml | 25 ++++++++++ 2 files changed, 75 insertions(+) create mode 100644 config/stage-a-o96-multi-step.yaml create mode 100644 resources/inference/configs/o96-global-multistep-forecaster.yaml diff --git a/config/stage-a-o96-multi-step.yaml b/config/stage-a-o96-multi-step.yaml new file mode 100644 index 00000000..5a817d63 --- /dev/null +++ b/config/stage-a-o96-multi-step.yaml @@ -0,0 +1,50 @@ +# yaml-language-server: $schema=../workflow/tools/config.schema.json +description: | + Evaluate skill of a stage A o96 multi-step global model against ERA. + +dates: + start: 2024-01-01T00:00 + end: 2024-01-01T00:00 + frequency: 24h + +runs: + - forecaster: + checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6/a689d740f37642c38fd01a39cdbde96f/inference-last.ckpt + label: stage-a-o96-multi-step + steps: 0/120/1 + config: resources/inference/configs/o96-global-multistep-forecaster.yaml + disable_local_eccodes_definitions: true + +truth: + label: ERA5-o96 + root: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-o96-1979-2024-1h-v3-with-era51.zarr + +experiment: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - TOT_PREC + dashboard: + stratification: + # - region + # - init_hour + - season + stratification: + regions: [] + +locations: + output_root: output/ + +profile: + executor: slurm + global_resources: + gpus: 16 + default_resources: + slurm_partition: "postproc" + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: "1h" + gpus: 0 + jobs: 50 diff --git a/resources/inference/configs/o96-global-multistep-forecaster.yaml b/resources/inference/configs/o96-global-multistep-forecaster.yaml new file mode 100644 index 00000000..8c7b6037 --- /dev/null +++ b/resources/inference/configs/o96-global-multistep-forecaster.yaml @@ -0,0 +1,25 @@ +input: + test: + use_original_paths: true + +allow_nans: true + +patch_metadata: + config: + dataloader: + test: + datasets: + data: + dataset_config: + dataset: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-o96-1979-2024-1h-v3-with-era51.zarr + +post_processors: + - accumulate_from_start_of_forecast: + accumulations: + - tp + +output: + grib: + path: grib/{date}{time:04}_{step:03}.grib + +write_initial_state: true From 48c98bb75eadee4dc844551944cafa02126ffbc5 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Tue, 30 Jun 2026 22:34:06 +0200 Subject: [PATCH 03/28] Allow for no baselines in meteograms --- config/stage-a-o96-multi-step.yaml | 2 +- workflow/rules/plot.smk | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config/stage-a-o96-multi-step.yaml b/config/stage-a-o96-multi-step.yaml index 5a817d63..3ff8ee23 100644 --- a/config/stage-a-o96-multi-step.yaml +++ b/config/stage-a-o96-multi-step.yaml @@ -14,7 +14,7 @@ runs: steps: 0/120/1 config: resources/inference/configs/o96-global-multistep-forecaster.yaml disable_local_eccodes_definitions: true - + truth: label: ERA5-o96 root: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-o96-1979-2024-1h-v3-with-era51.zarr diff --git a/workflow/rules/plot.smk b/workflow/rules/plot.smk index e51cbf51..fab9b1d1 100644 --- a/workflow/rules/plot.smk +++ b/workflow/rules/plot.smk @@ -12,6 +12,8 @@ import pandas as pd def _get_available_baselines(wc) -> list[dict[str, str]]: """Get all available baseline datasets for the given init time.""" baselines = [] + if not BASELINE_CONFIGS: + return baselines for baseline_id in BASELINE_CONFIGS: root = BASELINE_CONFIGS[baseline_id].get("root") steps = BASELINE_CONFIGS[baseline_id].get("steps") From 7032716a80e76090756268e18d16942dd7b26dc9 Mon Sep 17 00:00:00 2001 From: Michele Cattaneo Date: Wed, 1 Jul 2026 14:09:40 +0200 Subject: [PATCH 04/28] fix: fix global truth units for precip, to fix verification metrics --- src/data_input/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index ae343ca2..ea1d379a 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -349,6 +349,13 @@ def load_forecast_data_from_grib( if ifs_rename: ds = ds.rename(ifs_rename) + if "tp" in ifs_rename: + # IFS/ECMWF convention: "tp" is accumulated precip in meters. + # Convert to kg m-2 (mm) to match the ICON-native convention used + # elsewhere (truth-side conversion in load_analysis_data_from_zarr, + # and ICON-native forecast/truth pairs, which are already in mm). + ds["TOT_PREC"] = ds["TOT_PREC"] * 1000 + if "TOT_PREC" in ds.data_vars: ds["TOT_PREC"] = _tot_prec_handling(ds["TOT_PREC"], requested_steps=steps) From ec82cfd279722f9be6dac148cebaec40ceab9a60 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Thu, 2 Jul 2026 16:14:06 +0200 Subject: [PATCH 05/28] Use the rescaling postprocessor for handling unit conversions --- resources/inference/configs/aifs-single-forecaster.yaml | 5 +++++ .../inference/configs/o96-global-multistep-forecaster.yaml | 5 +++++ src/data_input/__init__.py | 7 ------- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/resources/inference/configs/aifs-single-forecaster.yaml b/resources/inference/configs/aifs-single-forecaster.yaml index f5003c13..51f236ef 100644 --- a/resources/inference/configs/aifs-single-forecaster.yaml +++ b/resources/inference/configs/aifs-single-forecaster.yaml @@ -12,6 +12,11 @@ post_processors: - accumulate_from_start_of_forecast: accumulations: - tp + - forward_transform_filter: + rescale: + scale: 1000 # convert units from m to kg m-2 + offset: 0 + param: tp output: grib: diff --git a/resources/inference/configs/o96-global-multistep-forecaster.yaml b/resources/inference/configs/o96-global-multistep-forecaster.yaml index 8c7b6037..dcef3e04 100644 --- a/resources/inference/configs/o96-global-multistep-forecaster.yaml +++ b/resources/inference/configs/o96-global-multistep-forecaster.yaml @@ -17,6 +17,11 @@ post_processors: - accumulate_from_start_of_forecast: accumulations: - tp + - forward_transform_filter: + rescale: + scale: 1000 # convert units from m to kg m-2 + offset: 0 + param: tp output: grib: diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index ea1d379a..ae343ca2 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -349,13 +349,6 @@ def load_forecast_data_from_grib( if ifs_rename: ds = ds.rename(ifs_rename) - if "tp" in ifs_rename: - # IFS/ECMWF convention: "tp" is accumulated precip in meters. - # Convert to kg m-2 (mm) to match the ICON-native convention used - # elsewhere (truth-side conversion in load_analysis_data_from_zarr, - # and ICON-native forecast/truth pairs, which are already in mm). - ds["TOT_PREC"] = ds["TOT_PREC"] * 1000 - if "TOT_PREC" in ds.data_vars: ds["TOT_PREC"] = _tot_prec_handling(ds["TOT_PREC"], requested_steps=steps) From d308e5c9a903bdef438cde43f8e3993eeedd5cc9 Mon Sep 17 00:00:00 2001 From: Michele Cattaneo Date: Fri, 3 Jul 2026 13:27:32 +0200 Subject: [PATCH 06/28] feat: support for cloud cover and rotating gifs --- config/stage-a-o96-multi-step.yaml | 43 ++++++++++++++++++++++--- src/data_input/__init__.py | 7 ++-- src/evalml/config.py | 20 ++++++++++++ src/plotting/__init__.py | 18 ++++------- src/plotting/colormap_defaults.py | 8 +++++ src/plotting/compat.py | 6 ++++ workflow/rules/common.smk | 25 ++++++++++---- workflow/scripts/plot_forecast_frame.py | 25 +++++++++++++- 8 files changed, 128 insertions(+), 24 deletions(-) diff --git a/config/stage-a-o96-multi-step.yaml b/config/stage-a-o96-multi-step.yaml index 3ff8ee23..c3c00ec0 100644 --- a/config/stage-a-o96-multi-step.yaml +++ b/config/stage-a-o96-multi-step.yaml @@ -3,18 +3,33 @@ description: | Evaluate skill of a stage A o96 multi-step global model against ERA. dates: - start: 2024-01-01T00:00 - end: 2024-01-01T00:00 - frequency: 24h + # start: 2024-01-01T00:00 + # end: 2024-07-01T00:00 + # frequency: 59h + - 2024-01-01T00:00 runs: - forecaster: checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6/a689d740f37642c38fd01a39cdbde96f/inference-last.ckpt - label: stage-a-o96-multi-step + label: resO96_fr1_st1_in7_out6 steps: 0/120/1 config: resources/inference/configs/o96-global-multistep-forecaster.yaml disable_local_eccodes_definitions: true + # - forecaster: + # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in2_out1/877cb9559f6c484d97b3733b7481c39a/inference-last.ckpt + # label: resO96_fr1_st1_in2_out1 + # steps: 0/120/1 + # config: resources/inference/configs/o96-global-multistep-forecaster.yaml + # disable_local_eccodes_definitions: true + + # - forecaster: + # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in4_out3/70b8ec9d55144f968513b706415ffc04/inference-last.ckpt + # label: resO96_fr1_st1_in4_out3 + # steps: 0/120/1 + # config: resources/inference/configs/o96-global-multistep-forecaster.yaml + # disable_local_eccodes_definitions: true + truth: label: ERA5-o96 root: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-o96-1979-2024-1h-v3-with-era51.zarr @@ -26,6 +41,7 @@ experiment: - U_10M - V_10M - TOT_PREC + - CLCT dashboard: stratification: # - region @@ -34,6 +50,25 @@ experiment: stratification: regions: [] +showcase: + params: + - T_2M + - SP_10M + - TOT_PREC + - CLCT + meteograms: + enabled: false # Because to Jretrieve credentials + animations: + enabled: true + domains: + - name: globe + rotate: true + hours_per_revolution: 120 + # - europe + # # - alps + # - icon-ch + # - switzerland + locations: output_root: output/ diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index ae343ca2..c74ac3f8 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -22,6 +22,7 @@ "2d": "TD_2M", "sp": "PS", "lsm": "FR_LAND", + "tcc": "CLCT", } _ICON_TO_IFS = {v: k for k, v in _IFS_TO_ICON.items()} @@ -216,10 +217,12 @@ def load_from_grib_file(file: str | list[str], sel_kwargs): def variable_name_profile( - level_type: Literal["height_above_ground_level", "mean_sea", "surface", "pressure"], + level_type: Literal[ + "height_above_ground_level", "mean_sea", "surface", "pressure", "entire_atmosphere" + ], ) -> dict[str, Any]: """Resolve variable name profile based on the level type.""" - if level_type in ["height_above_ground_level", "mean_sea", "surface"]: + if level_type in ["height_above_ground_level", "mean_sea", "surface", "entire_atmosphere"]: return {} elif level_type == "pressure": return { diff --git a/src/evalml/config.py b/src/evalml/config.py index cfc211ad..33d75d3c 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -272,9 +272,29 @@ class DomainConfig(BaseModel): "orthographic", description="Projection name (must be a key in plotting._PROJECTIONS, e.g. 'orthographic').", ) + rotate: bool = Field( + False, + description=( + "Rotate the viewpoint across animation frames as lead time advances. " + "Only valid for full-globe domains (extent: null)." + ), + ) + hours_per_revolution: float = Field( + 96.0, + gt=0, + description="Simulated lead-time hours for one full 360° rotation, when rotate is enabled.", + ) model_config = {"extra": "forbid"} + @model_validator(mode="after") + def _rotate_requires_globe(self): + if self.rotate and self.extent is not None: + raise ValueError( + "rotate: true is only valid for full-globe domains (extent: null)." + ) + return self + class MeteogramConfig(BaseModel): """Configuration for meteogram generation.""" diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index 524834c6..5e0fa2d1 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -1,5 +1,4 @@ from contextlib import contextmanager -from functools import cached_property from pathlib import Path import cartopy.crs as ccrs @@ -174,8 +173,8 @@ def plot_field( # of the plotting function is a lot faster than letting tricontourf or # tripcolor handle it in general, but not sure if using earthkit # removed for now to simplify the workflow - if proj == _PROJECTIONS["orthographic"]: - triang, mask = self._orthographic_tri + if isinstance(proj, ccrs.Orthographic): + triang, mask = self._orthographic_tri(proj) else: triang, mask = self.tri, slice(None, None) x, y = triang.x, triang.y @@ -265,13 +264,10 @@ def _temporary_plot_kwargs_override(self, subplot: ekp.Map): except Exception: pass - @cached_property - def _orthographic_tri(self) -> Triangulation: - """Compute the triangulation for the orthographic projection.""" - x, y, _ = ( - _PROJECTIONS["orthographic"] - .transform_points(ccrs.PlateCarree(), self.lon, self.lat) - .T - ) + def _orthographic_tri( + self, proj: ccrs.Projection + ) -> tuple[Triangulation, np.ndarray]: + """Compute the triangulation for an orthographic-family projection.""" + x, y, _ = proj.transform_points(ccrs.PlateCarree(), self.lon, self.lat).T mask = ~(np.isnan(x) | np.isnan(y)) return Triangulation(x[mask], y[mask]), mask diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index 1f1a4013..f23473e6 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -53,6 +53,14 @@ def _fallback(): "extend": "both", }, "QV_925": load_ncl_colormap("RH_6lev.ct") | {"extend": "both"}, + "CLCT": { + "cmap": plt.get_cmap("Blues", 10), + "vmin": 0, + "vmax": 1, + "extend": "neither", + "units": "", + "levels": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], + }, "TOT_PREC_1H": { "extend": "max", "colors": [ diff --git a/src/plotting/compat.py b/src/plotting/compat.py index f88dd51a..69db25b2 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -15,6 +15,7 @@ "PS": "sp", "PMSL": "msl", "TOT_PREC": "tp", + "CLCT": "tcc", } PARAMS_MAP_INV = {v: k for k, v in PARAMS_MAP.items()} @@ -35,6 +36,11 @@ def load_state_from_grib( } if ifs_rename: ds = ds.rename(ifs_rename) + # TODO check if needed + # if "tp" in ifs_rename and "TOT_PREC" in ds: + # # IFS/ECMWF convention: "tp" is accumulated precip in meters. + # # Convert to kg m-2 (mm) to match the ICON-native convention. + # ds["TOT_PREC"] = ds["TOT_PREC"] * 1000 state = {} ref_param = next((p for p in (paramlist or []) if p in ds), None) if ref_param is None: diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 4a4c29b8..e9b75b57 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -92,10 +92,12 @@ def parse_regions(): def parse_showcase_regions(): """Parse showcase domains from config. - Returns a dict mapping domain name -> {extent, projection}. + Returns a dict mapping domain name -> {extent, projection, rotate, hours_per_revolution}. Named domains (strings) have extent=None and projection=None, meaning the plot script will fall back to the DOMAINS lookup. Custom domains carry their explicit extent and projection. + rotate/hours_per_revolution only take effect when extent is None + (full-globe domains). """ result = {} for r in ( @@ -104,11 +106,18 @@ def parse_showcase_regions(): .get("domains", ["globe", "europe", "switzerland"]) ): if isinstance(r, str): - result[r] = {"extent": None, "projection": None} + result[r] = { + "extent": None, + "projection": None, + "rotate": False, + "hours_per_revolution": 96.0, + } else: result[r["name"]] = { "extent": r.get("extent"), "projection": r.get("projection", "orthographic"), + "rotate": r.get("rotate", False), + "hours_per_revolution": r.get("hours_per_revolution", 96.0), } return result @@ -392,10 +401,14 @@ SCORECARD_CONFIGS = ( ) -# Period-accumulated params verify a [lead - period, lead] window, so they have -# no value at lead times shorter than one step spacing (e.g. no 0h precip map). +# Params with no value at lead time 0. Two distinct reasons land here: +# - period-accumulated params (TOT_PREC/tp) verify a [lead - period, lead] +# window, so they have no value at lead times shorter than one step spacing +# (e.g. no 0h precip map). +# - diagnostic params (e.g. CLCT/tcc) aren't part of the model's input state, +# so they're simply absent from the initial-state GRIB file written at step 0. # Short and canonical names both appear across the workflow (showcases vs maps). -ACCUMULATED_PARAMS = {"TOT_PREC", "tp"} +PARAMS_WITHOUT_STEP_ZERO_VALUE = {"TOT_PREC", "tp", "CLCT", "tcc"} def resolve_leadtimes(steps_spec, requested="all", param=None): @@ -423,6 +436,6 @@ def resolve_leadtimes(steps_spec, requested="all", param=None): ) valid = wanted & supported - if param in ACCUMULATED_PARAMS: + if param in PARAMS_WITHOUT_STEP_ZERO_VALUE: valid = {lt for lt in valid if lt >= step} return sorted(valid) diff --git a/workflow/scripts/plot_forecast_frame.py b/workflow/scripts/plot_forecast_frame.py index 981f397e..f98a3b13 100644 --- a/workflow/scripts/plot_forecast_frame.py +++ b/workflow/scripts/plot_forecast_frame.py @@ -182,6 +182,16 @@ def main(): if region_cfg.get("extent") is not None: projection = get_projection(region_cfg.get("projection") or "orthographic") extent = region_cfg["extent"] + elif region_cfg.get("rotate"): + base = DOMAINS[region_name]["projection"].proj4_params + central_longitude = ( + base["lon_0"] + + 360.0 * lead_time / region_cfg["hours_per_revolution"] + ) % 360 + projection = ccrs.Orthographic( + central_longitude=central_longitude, central_latitude=base["lat_0"] + ) + extent = DOMAINS[region_name]["extent"] else: projection = DOMAINS[region_name]["projection"] extent = DOMAINS[region_name]["extent"] @@ -193,6 +203,14 @@ def main(): name=region_name, size=(6, 6), ) + if region_cfg.get("rotate"): + # earthkit.plots creates figures with constrained_layout=True + # (earthkit.plots.components.figures.Figure.__init__), which + # re-flows axes margins per draw to fit whichever gridline labels + # happen to be rendered — label content/width varies with rotation + # angle, shifting the globe within an otherwise fixed-size canvas. + # Freeze the layout so the map's position is identical every frame. + fig.fig.set_layout_engine(None) subplot = fig.add_map(row=0, column=0) plotter.plot_field( @@ -208,7 +226,12 @@ def main(): fig.title(f"{param}, time: {validtime}") outfn = outdir / f"frame_{lead_time}_{param}_{region_name}.png" - fig.save(outfn, bbox_inches="tight", dpi=200) + # earthkit.plots' Figure.save() defaults bbox_inches to "tight", which + # crops to each frame's own content extent — that extent varies with the + # rotating globe's gridline labels, making frames jump around when + # stitched into a GIF. Pass bbox_inches=None explicitly to override that + # default and always save the fixed full canvas. + fig.save(outfn, dpi=200, bbox_inches=None) LOG.info("saved: %s", outfn) From 5e749f624b37f868be7c4841d62409339c9be21e Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 3 Jul 2026 10:00:19 +0200 Subject: [PATCH 07/28] Document issue with missing coordinates --- src/data_input/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index c74ac3f8..e055ce12 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -127,7 +127,11 @@ def load_analysis_data_from_zarr( if "cell" in ds.dims: ds = ds.rename({"cell": "values"}) - # Drop grid points with undefined (NaN) coordinates + # Drop grid points with undefined (NaN) coordinates. This can occur when + # xarray opens a zarr dataset whose lat/lon arrays have fill_value=0.0: any + # grid point sitting exactly on 0° longitude is masked to NaN by xarray even + # though it is a valid point in the raw zarr (e.g. aifs-ea-an-oper o96 ERA5 + # dataset has 192 such points). if "values" in ds.dims and "latitude" in ds.coords and "longitude" in ds.coords: valid = np.isfinite(ds["latitude"].values) & np.isfinite(ds["longitude"].values) if not valid.all(): From acac66eceea1673d8bfc4bfb969f3cc0fe17ba47 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 3 Jul 2026 14:30:24 +0200 Subject: [PATCH 08/28] Add support flexible verification regions via config --- config/forecasters-ich1-oper-fixed.yaml | 1 + config/forecasters-ich1-oper.yaml | 1 + config/forecasters-ich1.yaml | 1 + config/varda-single-1.0.yaml | 1 + src/evalml/config.py | 31 +++++++++++++++++--- src/verification/__init__.py | 36 ++++++++++++++++-------- workflow/rules/common.smk | 29 +++++++++++++------ workflow/rules/verification.smk | 12 +++++--- workflow/scripts/verification_metrics.py | 18 ++++++++++-- workflow/tools/config.schema.json | 19 +++++++++++-- 10 files changed, 115 insertions(+), 34 deletions(-) diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index a4ef39f2..59d6390e 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -47,6 +47,7 @@ experiment: - TOT_PREC stratification: regions: + - all: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 9a0b33b2..4fa9d1fd 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -45,6 +45,7 @@ experiment: - TOT_PREC stratification: regions: + - all: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index d5dbf4ed..3f4f380a 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -57,6 +57,7 @@ experiment: - TOT_PREC stratification: regions: + - all: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/varda-single-1.0.yaml b/config/varda-single-1.0.yaml index f39000fe..aa0e413d 100644 --- a/config/varda-single-1.0.yaml +++ b/config/varda-single-1.0.yaml @@ -53,6 +53,7 @@ experiment: - TOT_PREC stratification: regions: + - all: [1.5, 16, 43, 49.5] - mittelland - berge - alpennordseite diff --git a/src/evalml/config.py b/src/evalml/config.py index 33d75d3c..aa905191 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Dict, List, Any, ClassVar, FrozenSet, Optional +from typing import Dict, List, Any, ClassVar, FrozenSet, Optional, Union from pydantic import BaseModel, Field, RootModel, field_validator, model_validator @@ -396,15 +396,38 @@ class Locations(BaseModel): class Stratification(BaseModel): """Stratification settings for the analysis.""" - regions: List[str] = Field( + regions: List[Union[str, Dict[str, List[float]]]] = Field( default_factory=list, - description="List of region names for stratification. Empty list means no spatial stratification.", + description=( + "List of region specs for spatial stratification. String entries are shapefile names " + "(resolved against 'root'). Dict entries map a region name to a bounding box " + "[lon_min, lon_max, lat_min, lat_max]. The special key 'all' overrides the default " + "full-domain region; any other key adds a named bbox region." + ), ) root: Optional[str] = Field( None, - description="Root directory where the region shapefiles are stored. Required when regions is non-empty.", + description="Root directory where the region shapefiles are stored. Required when regions contains string entries.", ) + @field_validator("regions") + @classmethod + def validate_regions( + cls, v: List[Union[str, Dict[str, List[float]]]] + ) -> List[Union[str, Dict[str, List[float]]]]: + for entry in v: + if isinstance(entry, dict): + if len(entry) != 1: + raise ValueError( + f"Each bbox region dict must have exactly one key, got: {list(entry.keys())}" + ) + name, bbox = next(iter(entry.items())) + if len(bbox) != 4: + raise ValueError( + f"Bbox for region '{name}' must have exactly 4 values [lon_min, lon_max, lat_min, lat_max], got {len(bbox)}." + ) + return v + class Dashboard(BaseModel): """Settings for the dashboard""" diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 7f3356da..343ce268 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -44,13 +44,9 @@ def __init__( src_crs.proj4_init, dst_crs.proj4_init, always_xy=True ).transform - regions = {} + regions = {"all": None} has_shapefiles = bool(shp and shp != [""]) if has_shapefiles: - # With explicit regional shapefiles, restrict "all" to the Alpine inner domain - regions["all"] = [ - Polygon(list(zip([1.5, 16, 16, 1.5, 1.5], [43, 43, 49.5, 49.5, 43]))) - ] shp = [shp] if isinstance(shp, str) else shp for shapefile in shp: region_name = Path(shapefile).stem @@ -58,9 +54,6 @@ def __init__( regions[region_name] = [ transform(proj, record.geometry) for record in reader.records() ] - else: - # No shapefile regions: "all" covers the full domain (e.g. global evaluation) - regions["all"] = None self.regions = regions def get_masks(self, lat: xr.DataArray, lon: xr.DataArray) -> xr.DataArray: @@ -235,12 +228,25 @@ def _merge_metrics(ds: xr.Dataset, num_workers: int = 4) -> xr.Dataset: return out +def _bbox_polygon(lon_min, lon_max, lat_min, lat_max) -> Polygon: + return Polygon( + [ + (lon_min, lat_min), + (lon_max, lat_min), + (lon_max, lat_max), + (lon_min, lat_max), + (lon_min, lat_min), + ] + ) + + def verify( fcst: xr.Dataset, obs: xr.Dataset, fcst_label: str, obs_label: str, - regions: list[str] | None = None, + shp_regions: list[str] | None = None, + bbox_regions: dict[str, list[float]] | None = None, dim: list[str] | None = None, threshold_dict: dict[str, dict[str, list[float]]] | None = None, num_workers: int | None = None, @@ -262,8 +268,12 @@ def verify( Label for the forecast source (used in output dataset). obs_label : str Label for the observation source (used in output dataset). - regions : list[str] or None, optional - List of shapefile paths or region names to use for spatial aggregation. If None, uses default region ('all'). + shp_regions : list[str] or None, optional + List of shapefile paths for spatial stratification. Region names are taken from the file stems. + bbox_regions : dict[str, list[float]] or None, optional + Named bounding-box regions as ``{name: [lon_min, lon_max, lat_min, lat_max]}``. + The ``"all"`` key overrides the default full-domain region. + Any other key adds an additional named region. dim : list[str] or None, optional List of dimension names to reduce over when computing metrics/statistics. If None, tries to infer from fcst. threshold_dict : dict[str, dict[str, list[float]]] or None, optional @@ -296,7 +306,9 @@ def verify( dim = ["values"] fcst_aligned, obs_aligned = xr.align(fcst, obs, join="inner", copy=False) - region_polygons = ShapefileSpatialAggregationMasks(shp=regions) + region_polygons = ShapefileSpatialAggregationMasks(shp=shp_regions or []) + for name, bbox in (bbox_regions or {}).items(): + region_polygons.regions[name] = [_bbox_polygon(*bbox)] masks = region_polygons.get_masks( lon=obs_aligned["longitude"], lat=obs_aligned["latitude"] ) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index cdd225ab..b3906f45 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -79,14 +79,26 @@ def parse_reference_times(): return times -def parse_regions(): - """Parse regions from the configuration.""" +def parse_shp_regions(): + """Return comma-separated shapefile paths from the regions config.""" cfg = config["experiment"]["stratification"] - region_names = cfg.get("regions", []) - if not region_names: - return "" - regions = [f"{cfg['root']}/{region}.shp" for region in region_names] - return ",".join(regions) + root = cfg.get("root", "") + return ",".join( + f"{root}/{entry}.shp" + for entry in cfg.get("regions", []) + if isinstance(entry, str) + ) + + +def parse_bbox_regions(): + """Return semicolon-separated name:lon_min,lon_max,lat_min,lat_max entries.""" + cfg = config["experiment"]["stratification"] + parts = [] + for entry in cfg.get("regions", []): + if isinstance(entry, dict): + for name, bbox in entry.items(): + parts.append(f"{name}:{','.join(str(v) for v in bbox)}") + return ";".join(parts) def parse_showcase_regions(): @@ -385,7 +397,8 @@ if "jretrieve" in str(config["truth"]["root"]): TRUTH_HASH = truth_hash(config["truth"]) -REGIONS = parse_regions() +SHP_REGIONS = parse_shp_regions() +BBOX_REGIONS = parse_bbox_regions() SHOWCASE_REGIONS = parse_showcase_regions() SHOWCASE_PARAMS = config.get("showcase", {}).get("params", ["T_2M", "SP_10M"]) EXPERIMENT_PARAMS = config.get("experiment", {}).get( diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index fe4b72d0..c4a48a70 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -29,7 +29,8 @@ rule verification_metrics_baseline: member=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("member", "000"), truth=config["truth"]["root"], truth_source_id=f"truth-{TRUTH_HASH}", - regions=REGIONS, + shp_regions=SHP_REGIONS, + bbox_regions=BBOX_REGIONS, experiment_params=",".join(EXPERIMENT_PARAMS), threshold_dict=config["experiment"]["thresholds"], shell: @@ -42,7 +43,8 @@ rule verification_metrics_baseline: --steps "{params.baseline_steps}" \ --source_id "{wildcards.baseline_id}" \ --truth_source_id "{params.truth_source_id}" \ - --regions "{params.regions}" \ + --shp_regions "{params.shp_regions}" \ + --bbox_regions "{params.bbox_regions}" \ --params "{params.experiment_params}" \ --threshold_dict "{params.threshold_dict}" \ --member "{params.member}" \ @@ -79,7 +81,8 @@ rule verification_metrics: fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], truth=config["truth"]["root"], truth_source_id=f"truth-{TRUTH_HASH}", - regions=REGIONS, + shp_regions=SHP_REGIONS, + bbox_regions=BBOX_REGIONS, grib_out_dir=lambda wc: ( Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" ).resolve(), @@ -95,7 +98,8 @@ rule verification_metrics: --steps "{params.fcst_steps}" \ --source_id "{wildcards.run_id}" \ --truth_source_id "{params.truth_source_id}" \ - --regions "{params.regions}" \ + --shp_regions "{params.shp_regions}" \ + --bbox_regions "{params.bbox_regions}" \ --params "{params.experiment_params}" \ --threshold_dict "{params.threshold_dict}" \ --output {output} >{log} 2>&1 diff --git a/workflow/scripts/verification_metrics.py b/workflow/scripts/verification_metrics.py index bbe7fd22..56b870d2 100644 --- a/workflow/scripts/verification_metrics.py +++ b/workflow/scripts/verification_metrics.py @@ -91,7 +91,8 @@ def main(args: ScriptConfig): truth, args.source_id, args.truth_source_id, - args.regions, + shp_regions=args.shp_regions, + bbox_regions=args.bbox_regions, threshold_dict=args.threshold_dict, ) LOG.info( @@ -158,9 +159,20 @@ def main(args: ScriptConfig): help="Stable identifier for the truth source (e.g. truth_).", ) parser.add_argument( - "--regions", + "--shp_regions", type=lambda x: [r for r in x.split(",") if r], - help="Comma-separated list of shapefile paths defining regions for stratification.", + help="Comma-separated list of shapefile paths for spatial stratification.", + default="", + ) + parser.add_argument( + "--bbox_regions", + type=lambda x: { + name: [float(v) for v in bbox.split(",")] + for part in x.split(";") + if part + for name, bbox in [part.split(":", 1)] + }, + help="Semicolon-separated bounding-box regions as name:lon_min,lon_max,lat_min,lat_max.", default="", ) parser.add_argument( diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index b974e30e..9a71fbc5 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -752,9 +752,22 @@ "description": "Stratification settings for the analysis.", "properties": { "regions": { - "description": "List of region names for stratification. Empty list means no spatial stratification.", + "description": "List of region specs for spatial stratification. String entries are shapefile names (resolved against 'root'). Dict entries map a region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. The special key 'all' overrides the default full-domain region; any other key adds a named bbox region.", "items": { - "type": "string" + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": { + "items": { + "type": "number" + }, + "type": "array" + }, + "type": "object" + } + ] }, "title": "Regions", "type": "array" @@ -769,7 +782,7 @@ } ], "default": null, - "description": "Root directory where the region shapefiles are stored. Required when regions is non-empty.", + "description": "Root directory where the region shapefiles are stored. Required when regions contains string entries.", "title": "Root" } }, From ff74be91730f2c4704b8593c451b7ad77be37c79 Mon Sep 17 00:00:00 2001 From: Michele Cattaneo Date: Fri, 3 Jul 2026 15:23:34 +0200 Subject: [PATCH 09/28] fix: fix bug for contorf plot for CLCT --- src/plotting/colormap_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index f23473e6..a11b939a 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -57,7 +57,7 @@ def _fallback(): "cmap": plt.get_cmap("Blues", 10), "vmin": 0, "vmax": 1, - "extend": "neither", + "extend": "both", "units": "", "levels": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], }, From 3b4b240b21aaee029d124f9bab996559082a8425 Mon Sep 17 00:00:00 2001 From: Michele Cattaneo Date: Fri, 3 Jul 2026 15:41:25 +0200 Subject: [PATCH 10/28] fix: regenerated config JSON schema for new animation params --- workflow/tools/config.schema.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 9a71fbc5..97c2d934 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -227,6 +227,19 @@ "description": "Projection name (must be a key in plotting._PROJECTIONS, e.g. 'orthographic').", "title": "Projection", "type": "string" + }, + "rotate": { + "default": false, + "description": "Rotate the viewpoint across animation frames as lead time advances. Only valid for full-globe domains (extent: null).", + "title": "Rotate", + "type": "boolean" + }, + "hours_per_revolution": { + "default": 96.0, + "description": "Simulated lead-time hours for one full 360\u00b0 rotation, when rotate is enabled.", + "exclusiveMinimum": 0, + "title": "Hours Per Revolution", + "type": "number" } }, "required": [ From 1ef6528870326ef0329115ed871331f933de419c Mon Sep 17 00:00:00 2001 From: Michele Cattaneo Date: Mon, 13 Jul 2026 15:56:06 +0200 Subject: [PATCH 11/28] chore: added support for plotting radiation and fixed plotting bug for cloud cover Co-authored-by: clairemerker <34312518+clairemerker@users.noreply.github.com> --- src/data_input/__init__.py | 8 +++++++ src/plotting/__init__.py | 15 ++++++++++++- src/plotting/colormap_defaults.py | 30 ++++++++++++++++++++++--- src/plotting/compat.py | 2 ++ workflow/rules/common.smk | 11 ++++++++- workflow/scripts/plot_forecast_frame.py | 27 +++++++++++++--------- 6 files changed, 77 insertions(+), 16 deletions(-) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index e055ce12..dbfd79e3 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -23,6 +23,14 @@ "sp": "PS", "lsm": "FR_LAND", "tcc": "CLCT", + "lcc": "CLCL", + # TODO: ssrd is treated as a plain per-step field (no de-accumulation), + # which only holds because it's not currently listed in any + # accumulate_from_start_of_forecast.accumulations in the inference + # configs (unlike tp, see _tot_prec_handling). If ssrd/strd are ever + # added there, this needs the same cumulative-since-start handling tp + # gets, or verification/plots will silently be wrong. + "ssrd": "SSRD", } _ICON_TO_IFS = {v: k for k, v in _IFS_TO_ICON.items()} diff --git a/src/plotting/__init__.py b/src/plotting/__init__.py index 5e0fa2d1..63b6be12 100644 --- a/src/plotting/__init__.py +++ b/src/plotting/__init__.py @@ -147,6 +147,7 @@ def plot_field( style: ekp.styles.Style | None = None, colorbar: bool = True, title: str | None = None, + gridline_labels: bool = True, **kwargs, ): """Plot a field on a Map object. @@ -163,6 +164,12 @@ def plot_field( Whether to plot a colorbar, by default True. title: str, optional Map subplot title. + gridline_labels : bool + Whether to draw lat/lon degree labels on the gridlines, by + default True. Set to False for views whose center rotates + between frames (e.g. a rotating globe animation), where the + labels' varying width would otherwise make the map's position + shift from frame to frame. kwargs : dict Additional keyword arguments to pass to ax.tripcolor, including cmap, vmin, vmax, etc. @@ -214,7 +221,13 @@ def plot_field( # TODO: gridlines etc would be nicer to have in the init, but I didn't get # them to overlay the plot layer - subplot.standard_layers() + if gridline_labels: + subplot.standard_layers() + else: + subplot.land() + subplot.coastlines() + subplot.borders() + subplot.gridlines(draw_labels=False) if colorbar: subplot.legend() diff --git a/src/plotting/colormap_defaults.py b/src/plotting/colormap_defaults.py index a11b939a..9ab83a99 100644 --- a/src/plotting/colormap_defaults.py +++ b/src/plotting/colormap_defaults.py @@ -54,12 +54,36 @@ def _fallback(): }, "QV_925": load_ncl_colormap("RH_6lev.ct") | {"extend": "both"}, "CLCT": { - "cmap": plt.get_cmap("Blues", 10), + # extend="neither" relies on preprocess_field() clipping away from + # exact 0/1 (see plot_forecast_frame.py) to avoid a tricontourf bug + # on orthographic projections. + "cmap": plt.get_cmap("Blues_r"), "vmin": 0, "vmax": 1, - "extend": "both", + "extend": "neither", + "units": "", + "levels": list(np.linspace(0, 1, 21)), + }, + "CLCL": { + "cmap": plt.get_cmap("Blues_r"), + "vmin": 0, + "vmax": 1, + "extend": "neither", "units": "", - "levels": [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0], + "levels": list(np.linspace(0, 1, 21)), + }, + "SSRD": { + # tricontourf always bands regardless of "levels" being set (it falls + # back to an auto locator with ~7 bands otherwise) — use a fine level + # set here to approximate a smooth gradient instead. extend="max" + # only (not "both") since preprocess_field() already clips away from + # exact 0 — see CLCT. + "cmap": plt.get_cmap("YlOrRd"), + "vmin": 0, + "vmax": 4e6, + "extend": "max", + "units": "J m-2", + "levels": list(np.linspace(0, 4e6, 21)), }, "TOT_PREC_1H": { "extend": "max", diff --git a/src/plotting/compat.py b/src/plotting/compat.py index 69db25b2..caf33be9 100644 --- a/src/plotting/compat.py +++ b/src/plotting/compat.py @@ -16,6 +16,8 @@ "PMSL": "msl", "TOT_PREC": "tp", "CLCT": "tcc", + "CLCL": "lcc", + "SSRD": "ssrd", } PARAMS_MAP_INV = {v: k for k, v in PARAMS_MAP.items()} diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index b3906f45..aeb1973d 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -422,7 +422,16 @@ SCORECARD_CONFIGS = ( # - diagnostic params (e.g. CLCT/tcc) aren't part of the model's input state, # so they're simply absent from the initial-state GRIB file written at step 0. # Short and canonical names both appear across the workflow (showcases vs maps). -PARAMS_WITHOUT_STEP_ZERO_VALUE = {"TOT_PREC", "tp", "CLCT", "tcc"} +PARAMS_WITHOUT_STEP_ZERO_VALUE = { + "TOT_PREC", + "tp", + "CLCT", + "tcc", + "CLCL", + "lcc", + "SSRD", + "ssrd", +} def resolve_leadtimes(steps_spec, requested="all", param=None): diff --git a/workflow/scripts/plot_forecast_frame.py b/workflow/scripts/plot_forecast_frame.py index f98a3b13..c117c779 100644 --- a/workflow/scripts/plot_forecast_frame.py +++ b/workflow/scripts/plot_forecast_frame.py @@ -88,6 +88,15 @@ def preprocess_field(param: str, state: dict): return ekm_wind.speed(fields["U"], fields["V"]), "m/s" if param == "TOT_PREC": return np.maximum(fields[param], 0), "mm" + if param in ("CLCT", "CLCL"): + # Avoid exact 0/1 plateaus breaking tricontourf on orthographic + # projections (tmp/reproduce_clct_bug.py). Pair with extend="neither". + # Any new bounded field with silent-blank or GeometryCollection-crash + # globe frames likely needs the same clip-away-from-boundary fix. + return np.clip(fields[param], 1e-6, 1 - 1e-6), None + if param == "SSRD": + # Same issue, bottom boundary only (night-side plateau). + return np.maximum(fields[param], 1e-6), None return fields[param], None @@ -178,6 +187,7 @@ def main(): for region_name, region_cfg in regions.items(): LOG.info("Plotting region %s", region_name) + outfn = outdir / f"frame_{lead_time}_{param}_{region_name}.png" plotter = StatePlotter(state["longitudes"], state["latitudes"], outdir) if region_cfg.get("extent") is not None: projection = get_projection(region_cfg.get("projection") or "orthographic") @@ -188,6 +198,7 @@ def main(): base["lon_0"] + 360.0 * lead_time / region_cfg["hours_per_revolution"] ) % 360 + # central_longitude=central_longitude, central_latitude=0.0 for zero-centered projection = ccrs.Orthographic( central_longitude=central_longitude, central_latitude=base["lat_0"] ) @@ -203,18 +214,14 @@ def main(): name=region_name, size=(6, 6), ) - if region_cfg.get("rotate"): - # earthkit.plots creates figures with constrained_layout=True - # (earthkit.plots.components.figures.Figure.__init__), which - # re-flows axes margins per draw to fit whichever gridline labels - # happen to be rendered — label content/width varies with rotation - # angle, shifting the globe within an otherwise fixed-size canvas. - # Freeze the layout so the map's position is identical every frame. - fig.fig.set_layout_engine(None) subplot = fig.add_map(row=0, column=0) plotter.plot_field( - subplot, field, **get_style(param, units_override, accu=accu) + subplot, + field, + title=f"{param}, time: {validtime}", + gridline_labels=not region_cfg.get("rotate", False), + **get_style(param, units_override, accu=accu), ) if len(state["lam_envelope"]) > 0: subplot.ax.add_geometries( @@ -223,9 +230,7 @@ def main(): facecolor="none", crs=ccrs.PlateCarree(), ) - fig.title(f"{param}, time: {validtime}") - outfn = outdir / f"frame_{lead_time}_{param}_{region_name}.png" # earthkit.plots' Figure.save() defaults bbox_inches to "tight", which # crops to each frame's own content extent — that extent varies with the # rotating globe's gridline labels, making frames jump around when From 7da90678bfafba8c42bb40091d54d7f2d9826e73 Mon Sep 17 00:00:00 2001 From: Michele Cattaneo Date: Mon, 13 Jul 2026 16:10:43 +0200 Subject: [PATCH 12/28] chore: added n320 global config --- config/stage-a-o96-multi-step.yaml | 15 ++- config/stage-b-n320-multi-step.yaml | 95 +++++++++++++++++++ .../n320-global-multistep-forecaster.yaml | 31 ++++++ 3 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 config/stage-b-n320-multi-step.yaml create mode 100644 resources/inference/configs/n320-global-multistep-forecaster.yaml diff --git a/config/stage-a-o96-multi-step.yaml b/config/stage-a-o96-multi-step.yaml index c3c00ec0..7d41bdb2 100644 --- a/config/stage-a-o96-multi-step.yaml +++ b/config/stage-a-o96-multi-step.yaml @@ -3,10 +3,10 @@ description: | Evaluate skill of a stage A o96 multi-step global model against ERA. dates: - # start: 2024-01-01T00:00 - # end: 2024-07-01T00:00 - # frequency: 59h - - 2024-01-01T00:00 + start: 2024-01-01T00:00 + end: 2024-07-01T00:00 + frequency: 59h + # - 2024-01-01T00:00 runs: - forecaster: @@ -16,6 +16,13 @@ runs: config: resources/inference/configs/o96-global-multistep-forecaster.yaml disable_local_eccodes_definitions: true + # - forecaster: + # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6/f812c442ebee498485da0f7c4a74bb7a/inference-last.ckpt + # label: resO96_fr1_st1_in7_out6_w_1_1_2_2_4_8 + # steps: 0/120/1 + # config: resources/inference/configs/o96-global-multistep-forecaster.yaml + # disable_local_eccodes_definitions: true + # - forecaster: # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in2_out1/877cb9559f6c484d97b3733b7481c39a/inference-last.ckpt # label: resO96_fr1_st1_in2_out1 diff --git a/config/stage-b-n320-multi-step.yaml b/config/stage-b-n320-multi-step.yaml new file mode 100644 index 00000000..d84492c5 --- /dev/null +++ b/config/stage-b-n320-multi-step.yaml @@ -0,0 +1,95 @@ +# yaml-language-server: $schema=../workflow/tools/config.schema.json +description: | + Evaluate skill of a stage A n320 multi-step global model against ERA. + +dates: + start: 2024-01-01T00:00 + end: 2024-07-01T00:00 + frequency: 59h + # - 2024-01-01T00:00 + # - 2024-06-19T00:00 + +runs: + - forecaster: + checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in7_out6/a4bae159c2e64af6a749031737bee02e/inference-last.ckpt + label: resN320_fr1_st1_in7_out6 + steps: 0/120/1 + config: resources/inference/configs/n320-global-multistep-forecaster.yaml + disable_local_eccodes_definitions: true + + # - forecaster: + # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in7_out6_rl4/482d4dfa299143e996ec2f6962510738/inference-anemoi-by_step-epoch_005-step_012000.ckpt + # label: resN320_fr1_st1_in7_out6_rl4 + # steps: 0/120/1 + # config: resources/inference/configs/n320-global-multistep-forecaster.yaml + # disable_local_eccodes_definitions: true + + # - forecaster: + # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in2_out1/406bb2ab927e4f7eb1f5c133b53ddcbc/inference-last.ckpt + # label: resN320_fr1_st1_in2_out1 + # steps: 0/120/1 + # config: resources/inference/configs/n320-global-multistep-forecaster.yaml + # disable_local_eccodes_definitions: true + + # - forecaster: + # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in4_out3/f64c33bccca74411bb267b02df8723e5/inference-last.ckpt + # label: resN320_fr1_st1_in4_out3 + # steps: 0/120/1 + # config: resources/inference/configs/n320-global-multistep-forecaster.yaml + # disable_local_eccodes_definitions: true + +truth: + label: ERA5-n320 + root: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-n320-1979-2024-1h-v2-with-era51.zarr + +experiment: + params: + - T_2M + - TD_2M + - U_10M + - V_10M + - TOT_PREC + - CLCT + dashboard: + stratification: + # - region + # - init_hour + - season + stratification: + regions: [] + +showcase: + params: + - T_2M + - SP_10M + - TOT_PREC + - CLCT + - CLCL + - SSRD + meteograms: + enabled: false # Because to Jretrieve credentials + animations: + enabled: true + domains: + - name: globe + rotate: false + hours_per_revolution: 240 # twice the horizon of 120h (it makes half a revolution) + # - europe + # # - alps + # - icon-ch + # - switzerland + +locations: + output_root: output/ + +profile: + executor: slurm + global_resources: + gpus: 16 + default_resources: + slurm_partition: "postproc" + cpus_per_task: 1 + mem_mb_per_cpu: 1800 + runtime: "1h" + gpus: 0 + jobs: 50 diff --git a/resources/inference/configs/n320-global-multistep-forecaster.yaml b/resources/inference/configs/n320-global-multistep-forecaster.yaml new file mode 100644 index 00000000..c94ad776 --- /dev/null +++ b/resources/inference/configs/n320-global-multistep-forecaster.yaml @@ -0,0 +1,31 @@ +input: + test: + use_original_paths: true + +allow_nans: true + +patch_metadata: + config: + dataloader: + test: + datasets: + data: + dataset_config: + dataset: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-n320-1979-2024-1h-v2-with-era51.zarr + + +post_processors: + - accumulate_from_start_of_forecast: + accumulations: + - tp + - forward_transform_filter: + rescale: + scale: 1000 # convert units from m to kg m-2 + offset: 0 + param: tp + +output: + grib: + path: grib/{date}{time:04}_{step:03}.grib + +write_initial_state: true From 8227668287e31287f078c82bd147751567b2222f Mon Sep 17 00:00:00 2001 From: Michele Cattaneo Date: Fri, 17 Jul 2026 17:45:02 +0200 Subject: [PATCH 13/28] chore: added ssrd in experiment configs --- config/stage-a-o96-multi-step.yaml | 8 ++++++++ .../configs/n320-global-multistep-forecaster.yaml | 5 +++++ .../configs/o96-global-multistep-forecaster.yaml | 5 +++++ 3 files changed, 18 insertions(+) diff --git a/config/stage-a-o96-multi-step.yaml b/config/stage-a-o96-multi-step.yaml index 7d41bdb2..4ffd5640 100644 --- a/config/stage-a-o96-multi-step.yaml +++ b/config/stage-a-o96-multi-step.yaml @@ -15,6 +15,12 @@ runs: steps: 0/120/1 config: resources/inference/configs/o96-global-multistep-forecaster.yaml disable_local_eccodes_definitions: true + - forecaster: + checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6_water/b1b457100bbe42358bfab53fda8f6d67/inference-last.ckpt + label: resO96_fr1_st1_in7_out6_water + steps: 0/120/1 + config: resources/inference/configs/o96-global-multistep-forecaster.yaml + disable_local_eccodes_definitions: true # - forecaster: # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6/f812c442ebee498485da0f7c4a74bb7a/inference-last.ckpt @@ -49,6 +55,8 @@ experiment: - V_10M - TOT_PREC - CLCT + - CLCL + - SSRD dashboard: stratification: # - region diff --git a/resources/inference/configs/n320-global-multistep-forecaster.yaml b/resources/inference/configs/n320-global-multistep-forecaster.yaml index c94ad776..b3a076b9 100644 --- a/resources/inference/configs/n320-global-multistep-forecaster.yaml +++ b/resources/inference/configs/n320-global-multistep-forecaster.yaml @@ -27,5 +27,10 @@ post_processors: output: grib: path: grib/{date}{time:04}_{step:03}.grib + # ssrd/strd have no valid value at step 0 (1h-period accumulation ending + # at the current time would need data before the forecast start) — + # "skip" omits them from the initial state instead of erroring or writing + # a physically meaningless value. + negative_step_mode: skip write_initial_state: true diff --git a/resources/inference/configs/o96-global-multistep-forecaster.yaml b/resources/inference/configs/o96-global-multistep-forecaster.yaml index dcef3e04..ec9b9b79 100644 --- a/resources/inference/configs/o96-global-multistep-forecaster.yaml +++ b/resources/inference/configs/o96-global-multistep-forecaster.yaml @@ -26,5 +26,10 @@ post_processors: output: grib: path: grib/{date}{time:04}_{step:03}.grib + # ssrd/strd have no valid value at step 0 (1h-period accumulation ending + # at the current time would need data before the forecast start) — + # "skip" omits them from the initial state instead of erroring or writing + # a physically meaningless value. + negative_step_mode: skip write_initial_state: true From f99b4b1d76b69abad2ed03661ddfb7dec4061a36 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Tue, 21 Jul 2026 13:39:18 +0200 Subject: [PATCH 14/28] Fix region parsing --- workflow/rules/common.smk | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index cea26ab4..527483a8 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -416,7 +416,6 @@ TRUTH_HASH = truth_hash(config["truth"]) SHP_REGIONS = parse_shp_regions() BBOX_REGIONS = parse_bbox_regions() VERIF_HASH = verif_hash(config) -REGIONS = parse_regions() _showcase = config.get("showcase", {}) SHOWCASE_CONFIG = { "regions": parse_showcase_regions(), From 6d27edc31026977a941c21ce72eeb665f5ae92a3 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Tue, 21 Jul 2026 13:40:21 +0200 Subject: [PATCH 15/28] Linting --- src/data_input/__init__.py | 13 +++++++++++-- src/verification/__init__.py | 1 - workflow/scripts/plot_forecast_frame.py | 3 +-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index cbc7f571..f99915b2 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -484,11 +484,20 @@ def load_from_grib_file(file: str | list[str], sel_kwargs): def variable_name_profile( level_type: Literal[ - "height_above_ground_level", "mean_sea", "surface", "pressure", "entire_atmosphere" + "height_above_ground_level", + "mean_sea", + "surface", + "pressure", + "entire_atmosphere", ], ) -> dict[str, Any]: """Resolve variable name profile based on the level type.""" - if level_type in ["height_above_ground_level", "mean_sea", "surface", "entire_atmosphere"]: + if level_type in [ + "height_above_ground_level", + "mean_sea", + "surface", + "entire_atmosphere", + ]: return {} elif level_type == "pressure": return { diff --git a/src/verification/__init__.py b/src/verification/__init__.py index ae5716cb..3c342b12 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -121,7 +121,6 @@ def __init__( regions = {"all": None} has_shapefiles = bool(shp and shp != [""]) if has_shapefiles: - shp = [shp] if isinstance(shp, str) else shp for shapefile in shp: region_name = Path(shapefile).stem diff --git a/workflow/scripts/plot_forecast_frame.py b/workflow/scripts/plot_forecast_frame.py index c117c779..ee8e564e 100644 --- a/workflow/scripts/plot_forecast_frame.py +++ b/workflow/scripts/plot_forecast_frame.py @@ -195,8 +195,7 @@ def main(): elif region_cfg.get("rotate"): base = DOMAINS[region_name]["projection"].proj4_params central_longitude = ( - base["lon_0"] - + 360.0 * lead_time / region_cfg["hours_per_revolution"] + base["lon_0"] + 360.0 * lead_time / region_cfg["hours_per_revolution"] ) % 360 # central_longitude=central_longitude, central_latitude=0.0 for zero-centered projection = ccrs.Orthographic( From 2eee400fd190ff254adb09d13cbf7edbb2bc5145 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 22 Jul 2026 11:45:38 +0200 Subject: [PATCH 16/28] Fix aifs-single --- config/aifs-single.yaml | 7 +++---- resources/inference/configs/aifs-single-forecaster.yaml | 4 +++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/aifs-single.yaml b/config/aifs-single.yaml index 3a806361..d3a75980 100644 --- a/config/aifs-single.yaml +++ b/config/aifs-single.yaml @@ -18,17 +18,16 @@ runs: config: resources/inference/configs/aifs-single-forecaster.yaml extra_requirements: - torch-geometric==2.4.0 - - anemoi-inference==0.6.3 + - anemoi-inference==0.11.1 - anemoi-models==0.5.0 - https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl disable_local_eccodes_definitions: true - inference_resources: - slurm_partition: preemptible - gpus: 1 truth: label: KENDA-CH1 root: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-v1.0.zarr +lapse_rate_correction: false + experiment: params: - T_2M diff --git a/resources/inference/configs/aifs-single-forecaster.yaml b/resources/inference/configs/aifs-single-forecaster.yaml index 51f236ef..ed40d2f2 100644 --- a/resources/inference/configs/aifs-single-forecaster.yaml +++ b/resources/inference/configs/aifs-single-forecaster.yaml @@ -1,4 +1,6 @@ -input: test +input: + test: + use_original_paths: true allow_nans: true From 4d3752edd34931bacd57fac73e4b9a7043f0537b Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 22 Jul 2026 13:45:43 +0200 Subject: [PATCH 17/28] Aesthetics --- workflow/Snakefile | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 7657d27f..67838e97 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -25,14 +25,18 @@ include: "rules/verif_obs.smk" # about workflow # ----------------------------------------------------- +CANDIDATES = collect_all_candidates() +BASELINES = collect_all_baselines() + RUN_CMD = Path(".evalml_snakemake_cmd.txt").read_text().strip() CONFIG_HASH = master_hash() WHEN = datetime.now().strftime("%Y%m%d") CONFIG_FILE = workflow.config_settings.configfiles[0] CONFIG_LABEL = config["config_label"] or CONFIG_FILE.stem EXPERIMENT_NAME = f"{WHEN}_{CONFIG_LABEL}_{CONFIG_HASH}" -CANDIDATES = collect_all_candidates() -BASELINES = collect_all_baselines() +CANDIDATE_LABELS = ", ".join(v.get("label", k) for k, v in CANDIDATES.items()) +BASELINE_LABELS = ", ".join(v.get("label", k) for k, v in BASELINES.items()) +GROUNDTRUTH_LABEL = config["truth"].get("label", config["truth"]["root"]) DATA_DIR = OUT_ROOT / "data" LOGS_DIR = OUT_ROOT / "logs" @@ -83,26 +87,19 @@ onstart: print() _hr() print(_c("🚀 EvalML workflow started", "1")) - print(_c(f" Time: {when_iso}", "90")) - print(_c(f" Snakemake: {RUN_CMD}", "90")) - print(_c(f" Workdir: {Path.cwd()}", "90")) - print(_c(f" Config: {CONFIG_FILE.name}", "90")) - print(_c(f" Experiment: {EXPERIMENT_NAME}", "90")) - print( - _c( - f" Candidates: {", ".join(v.get("label", k) for k, v in CANDIDATES.items())}", - "90", - ) - ) - print( - _c( - f" Baselines: {", ".join(v.get("label", k) for k, v in BASELINES.items())}", - "90", - ) - ) - print(_c(f" Data dir: {DATA_DIR}", "90")) - print(_c(f" Logs dir: {LOGS_DIR}", "90")) - print(_c(f" Results dir: {RESULTS_DIR}", "90")) + print(_c(f" Time: {when_iso}", "90")) + print(_c(f" Snakemake: {RUN_CMD}", "90")) + print(_c(f" Workdir: {Path.cwd()}", "90")) + print(_c(f" Config: {CONFIG_FILE.name}", "90")) + print(_c(f" Experiment: {EXPERIMENT_NAME}", "90")) + if CANDIDATES: + print(_c(f" Candidates: {CANDIDATE_LABELS}", "90")) + if BASELINES: + print(_c(f" Baselines: {BASELINE_LABELS}", "90")) + print(_c(f" Ground truth: {GROUNDTRUTH_LABEL}", "90")) + print(_c(f" Data dir: {DATA_DIR}", "90")) + print(_c(f" Logs dir: {LOGS_DIR}", "90")) + print(_c(f" Results dir: {RESULTS_DIR}", "90")) _hr() print() From 05657be2d6bc28440b038ed48f444c985032333c Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 22 Jul 2026 13:46:22 +0200 Subject: [PATCH 18/28] Verify aifs against era5 --- config/aifs-single.yaml | 7 ++++--- src/data_input/__init__.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/aifs-single.yaml b/config/aifs-single.yaml index d3a75980..9c574176 100644 --- a/config/aifs-single.yaml +++ b/config/aifs-single.yaml @@ -22,9 +22,10 @@ runs: - anemoi-models==0.5.0 - https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl disable_local_eccodes_definitions: true + truth: - label: KENDA-CH1 - root: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2025-1h-pl13-v1.0.zarr + label: ERA5-o96 + root: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-o96-1979-2024-1h-v3-with-era51.zarr lapse_rate_correction: false @@ -33,7 +34,7 @@ experiment: - T_2M - TD_2M - SP_10M - - TOT_PREC6 + - TOT_PREC dashboard: stratification: # - region diff --git a/src/data_input/__init__.py b/src/data_input/__init__.py index bad62945..074921e0 100644 --- a/src/data_input/__init__.py +++ b/src/data_input/__init__.py @@ -35,7 +35,7 @@ # added there, this needs the same cumulative-since-start handling tp # gets, or verification/plots will silently be wrong. "ssrd": "SSRD", - "z": "FSI", + "z": "FIS", } _ICON_TO_IFS = {v: k for k, v in _IFS_TO_ICON.items()} From 69099e22f60583d238ef757864697336e98f3b5a Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 22 Jul 2026 17:15:50 +0200 Subject: [PATCH 19/28] Make regions explicit --- README.md | 1 + config/aifs-single.yaml | 3 +- config/forecasters-ich1-oper-fixed.yaml | 2 +- config/forecasters-ich1-oper.yaml | 2 +- config/forecasters-ich1.yaml | 2 +- config/forecasters-ich1_mec_ffv2.yaml | 1 + config/varda-single-1.0.yaml | 2 +- src/evalml/config.py | 15 +++-- src/verification/__init__.py | 55 ++++++++++--------- workflow/rules/common.smk | 34 +++++------- workflow/rules/verification.smk | 12 ++-- .../scripts/report_experiment_dashboard.py | 2 +- workflow/scripts/report_scorecard.py | 5 +- workflow/scripts/verification_metrics.py | 27 ++++----- workflow/scripts/verification_plot_metrics.py | 4 +- workflow/tools/config.schema.json | 2 +- 16 files changed, 85 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 1111586a..8328fd4a 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ truth: experiment: stratification: regions: + - icon-ch1: [1.5, 16, 43, 49.5] # domain region — must be first - jura - mittelland - voralpen diff --git a/config/aifs-single.yaml b/config/aifs-single.yaml index 9c574176..96535b45 100644 --- a/config/aifs-single.yaml +++ b/config/aifs-single.yaml @@ -41,7 +41,8 @@ experiment: # - init_hour - season stratification: - regions: [] + regions: + - global: [-180, 180, -90, 90] locations: output_root: output/ diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index cdfc8968..6b487206 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -46,7 +46,7 @@ experiment: - TOT_PREC6 stratification: regions: - - all: [1.5, 16, 43, 49.5] + - icon-ch1: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index fcaa5efb..77f350f3 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -44,7 +44,7 @@ experiment: - TOT_PREC6 stratification: regions: - - all: [1.5, 16, 43, 49.5] + - icon-ch1: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 8b2c8ec4..61e164b1 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -57,7 +57,7 @@ experiment: - PMSL stratification: regions: - - all: [1.5, 16, 43, 49.5] + - icon-ch1: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1_mec_ffv2.yaml b/config/forecasters-ich1_mec_ffv2.yaml index d8993364..498c6ee3 100644 --- a/config/forecasters-ich1_mec_ffv2.yaml +++ b/config/forecasters-ich1_mec_ffv2.yaml @@ -44,6 +44,7 @@ experiment: - TOT_PREC stratification: regions: + - icon-ch1: [1.5, 16, 43, 49.5] - jura root: /store_new/mch/msopr/ml/regions/Prognoseregionen_LV95_20220517 thresholds: diff --git a/config/varda-single-1.0.yaml b/config/varda-single-1.0.yaml index c5a559d2..a13f3c3a 100644 --- a/config/varda-single-1.0.yaml +++ b/config/varda-single-1.0.yaml @@ -54,7 +54,7 @@ experiment: - PMSL stratification: regions: - - all: [1.5, 16, 43, 49.5] + - icon-ch1: [1.5, 16, 43, 49.5] - mittelland - berge - alpennordseite diff --git a/src/evalml/config.py b/src/evalml/config.py index f4703f17..162219b5 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -405,10 +405,11 @@ class Stratification(BaseModel): regions: List[Union[str, Dict[str, List[float]]]] = Field( default_factory=list, description=( - "List of region specs for spatial stratification. String entries are shapefile names " - "(resolved against 'root'). Dict entries map a region name to a bounding box " - "[lon_min, lon_max, lat_min, lat_max]. The special key 'all' overrides the default " - "full-domain region; any other key adds a named bbox region." + "List of region specs for spatial stratification. At least one region is required. " + "String entries are shapefile names (resolved against 'root'). Dict entries map a " + "region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. " + "The first entry is the domain region used by the dashboard when region stratification " + "is not active (e.g. {'global': [-180, 180, -90, 90]} or {'icon-ch1': [1.5, 16, 43, 49.5]})." ), ) root: Optional[str] = Field( @@ -421,6 +422,12 @@ class Stratification(BaseModel): def validate_regions( cls, v: List[Union[str, Dict[str, List[float]]]] ) -> List[Union[str, Dict[str, List[float]]]]: + if not v: + raise ValueError( + "At least one region must be specified. " + "Add a domain bbox as the first entry, e.g. regions: [{global: [-180, 180, -90, 90]}] " + "for global models or [{icon-ch1: [1.5, 16, 43, 49.5]}] for ICON-CH1." + ) for entry in v: if isinstance(entry, dict): if len(entry) != 1: diff --git a/src/verification/__init__.py b/src/verification/__init__.py index 3c342b12..d4fa444b 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -112,33 +112,31 @@ class ShapefileSpatialAggregationMasks(SpatialAggregationMasks): regions: dict[str, list[Polygon]] def __init__( - self, shp: str | list[str], src_crs=ccrs.epsg(2056), dst_crs=ccrs.PlateCarree() + self, + regions: list[dict], + src_crs=ccrs.epsg(2056), + dst_crs=ccrs.PlateCarree(), ): proj = pyproj.Transformer.from_crs( src_crs.proj4_init, dst_crs.proj4_init, always_xy=True ).transform - regions = {"all": None} - has_shapefiles = bool(shp and shp != [""]) - if has_shapefiles: - shp = [shp] if isinstance(shp, str) else shp - for shapefile in shp: - region_name = Path(shapefile).stem - reader = Reader(shapefile) - regions[region_name] = [ + self.regions = {} + for spec in regions: + name = spec["name"] + if spec["type"] == "bbox": + lon_min, lon_max, lat_min, lat_max = spec["bbox"] + self.regions[name] = [_bbox_polygon(lon_min, lon_max, lat_min, lat_max)] + elif spec["type"] == "shp": + reader = Reader(spec["path"]) + self.regions[name] = [ transform(proj, record.geometry) for record in reader.records() ] - self.regions = regions def get_masks(self, lat: xr.DataArray, lon: xr.DataArray) -> xr.DataArray: masks = [] for region_name, polygons in self.regions.items(): - if polygons is None: - mask = xr.DataArray( - np.ones(lon.shape, dtype=bool), coords=lon.coords, dims=lon.dims - ) - else: - mask = self._mask_from_polygons(polygons, lat, lon) + mask = self._mask_from_polygons(polygons, lat, lon) masks.append(mask.assign_coords(region=region_name)) return xr.concat(masks, dim="region") @@ -319,8 +317,7 @@ def verify( obs: xr.Dataset, fcst_label: str, obs_label: str, - shp_regions: list[str] | None = None, - bbox_regions: dict[str, list[float]] | None = None, + regions: list[dict] | None = None, dim: list[str] | None = None, threshold_dict: dict[str, dict[str, list[float]]] | None = None, num_workers: int | None = None, @@ -342,12 +339,12 @@ def verify( Label for the forecast source (used in output dataset). obs_label : str Label for the observation source (used in output dataset). - shp_regions : list[str] or None, optional - List of shapefile paths for spatial stratification. Region names are taken from the file stems. - bbox_regions : dict[str, list[float]] or None, optional - Named bounding-box regions as ``{name: [lon_min, lon_max, lat_min, lat_max]}``. - The ``"all"`` key overrides the default full-domain region. - Any other key adds an additional named region. + regions : list[dict] or None, optional + Ordered list of region specs. Each entry is either + ``{"type": "bbox", "name": ..., "bbox": [lon_min, lon_max, lat_min, lat_max]}`` or + ``{"type": "shp", "name": ..., "path": ...}``. The list order is preserved in the + output NetCDF region coordinate; the first entry is the domain region used by + dashboards and scorecards when region stratification is not active. dim : list[str] or None, optional List of dimension names to reduce over when computing metrics/statistics. If None, tries to infer from fcst. threshold_dict : dict[str, dict[str, list[float]]] or None, optional @@ -379,10 +376,14 @@ def verify( else: dim = ["values"] + if not regions: + raise ValueError( + "At least one region must be specified. " + "Provide an ordered list of region specs via the 'regions' argument." + ) + fcst_aligned, obs_aligned = xr.align(fcst, obs, join="inner", copy=False) - region_polygons = ShapefileSpatialAggregationMasks(shp=shp_regions or []) - for name, bbox in (bbox_regions or {}).items(): - region_polygons.regions[name] = [_bbox_polygon(*bbox)] + region_polygons = ShapefileSpatialAggregationMasks(regions=regions) masks = region_polygons.get_masks( lon=obs_aligned["longitude"], lat=obs_aligned["latitude"] ) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 527483a8..1e34f407 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -79,26 +79,23 @@ def parse_reference_times(): return times -def parse_shp_regions(): - """Return comma-separated shapefile paths from the regions config.""" - cfg = config["experiment"]["stratification"] - root = cfg.get("root", "") - return ",".join( - f"{root}/{entry}.shp" - for entry in cfg.get("regions", []) - if isinstance(entry, str) - ) +def parse_regions(): + """Return a JSON list of region specs in config order. - -def parse_bbox_regions(): - """Return semicolon-separated name:lon_min,lon_max,lat_min,lat_max entries.""" + Each entry is either ``{"type": "bbox", "name": ..., "bbox": [...]}`` + or ``{"type": "shp", "name": ..., "path": ...}``, preserving the original + order so the NetCDF region coordinate matches the config. + """ cfg = config["experiment"]["stratification"] - parts = [] + root = cfg.get("root", "") + result = [] for entry in cfg.get("regions", []): - if isinstance(entry, dict): - for name, bbox in entry.items(): - parts.append(f"{name}:{','.join(str(v) for v in bbox)}") - return ";".join(parts) + if isinstance(entry, str): + result.append({"type": "shp", "name": entry, "path": f"{root}/{entry}.shp"}) + elif isinstance(entry, dict): + name, bbox = next(iter(entry.items())) + result.append({"type": "bbox", "name": name, "bbox": bbox}) + return json.dumps(result) def parse_showcase_regions(): @@ -413,8 +410,7 @@ if "jretrieve" in str(config["truth"]["root"]): TRUTH_HASH = truth_hash(config["truth"]) -SHP_REGIONS = parse_shp_regions() -BBOX_REGIONS = parse_bbox_regions() +REGIONS = parse_regions() VERIF_HASH = verif_hash(config) _showcase = config.get("showcase", {}) SHOWCASE_CONFIG = { diff --git a/workflow/rules/verification.smk b/workflow/rules/verification.smk index ca1b9340..4886a5f1 100644 --- a/workflow/rules/verification.smk +++ b/workflow/rules/verification.smk @@ -29,8 +29,7 @@ rule verification_metrics_baseline: member=lambda wc: BASELINE_CONFIGS[wc.baseline_id].get("member", "000"), truth=config["truth"]["root"], truth_source_id=f"truth-{TRUTH_HASH}", - shp_regions=SHP_REGIONS, - bbox_regions=BBOX_REGIONS, + regions=REGIONS, experiment_params=",".join(EXPERIMENT_PARAMS), threshold_dict=config["experiment"]["thresholds"], lapse_rate_flag=( @@ -48,8 +47,7 @@ rule verification_metrics_baseline: --steps "{params.baseline_steps}" \ --source_id "{wildcards.baseline_id}" \ --truth_source_id "{params.truth_source_id}" \ - --shp_regions "{params.shp_regions}" \ - --bbox_regions "{params.bbox_regions}" \ + --regions '{params.regions}' \ --params "{params.experiment_params}" \ --threshold_dict "{params.threshold_dict}" \ --member "{params.member}" \ @@ -87,8 +85,7 @@ rule verification_metrics: fcst_steps=lambda wc: RUN_CONFIGS[wc.run_id]["steps"], truth=config["truth"]["root"], truth_source_id=f"truth-{TRUTH_HASH}", - shp_regions=SHP_REGIONS, - bbox_regions=BBOX_REGIONS, + regions=REGIONS, grib_out_dir=lambda wc: ( Path(OUT_ROOT) / f"data/runs/{wc.run_id}/{wc.init_time}/grib" ).resolve(), @@ -109,8 +106,7 @@ rule verification_metrics: --steps "{params.fcst_steps}" \ --source_id "{wildcards.run_id}" \ --truth_source_id "{params.truth_source_id}" \ - --shp_regions "{params.shp_regions}" \ - --bbox_regions "{params.bbox_regions}" \ + --regions '{params.regions}' \ --params "{params.experiment_params}" \ --threshold_dict "{params.threshold_dict}" \ {params.lapse_rate_flag} \ diff --git a/workflow/scripts/report_experiment_dashboard.py b/workflow/scripts/report_experiment_dashboard.py index 24fd0953..b8d3abf8 100644 --- a/workflow/scripts/report_experiment_dashboard.py +++ b/workflow/scripts/report_experiment_dashboard.py @@ -84,7 +84,7 @@ def main(args): # retain only rows relevant for the active stratifications stratification = args.stratification if "region" not in stratification: - df = df[df["region"] == "all"] + df = df[df["region"] == df["region"].unique()[0]] if "season" not in stratification: df = df[df["season"] == "all"] if "init_hour" not in stratification: diff --git a/workflow/scripts/report_scorecard.py b/workflow/scripts/report_scorecard.py index 68096ed9..93a4a8d6 100644 --- a/workflow/scripts/report_scorecard.py +++ b/workflow/scripts/report_scorecard.py @@ -25,7 +25,7 @@ # Sentinel values that select the "aggregate over all" slice for each # stratification dimension that is not the active stratification axis. -_STRAT_ALL_VALUES = {"region": "all", "season": "all", "init_hour": -999} +_STRAT_ALL_VALUES = {"season": "all", "init_hour": -999} DEFAULT_PLOT_CFG = { "rcparams": { @@ -228,6 +228,9 @@ def _load_relative_diff(cfg: dict) -> xr.Dataset: model_ds = xr.open_dataset(cfg["model"]["path"]) baseline_ds = xr.open_dataset(cfg["baseline"]["path"]) + if strat_dim != "region": + sel_coords["region"] = model_ds["region"].values[0] + for label, ds in [("model", model_ds), ("baseline", baseline_ds)]: if "n_samples" not in ds.data_vars: raise ValueError( diff --git a/workflow/scripts/verification_metrics.py b/workflow/scripts/verification_metrics.py index 73ff3e2f..85a79879 100644 --- a/workflow/scripts/verification_metrics.py +++ b/workflow/scripts/verification_metrics.py @@ -1,3 +1,4 @@ +import json import logging from argparse import ArgumentParser from argparse import Namespace @@ -94,8 +95,7 @@ def main(args: ScriptConfig): truth, args.source_id, args.truth_source_id, - shp_regions=args.shp_regions, - bbox_regions=args.bbox_regions, + regions=args.regions, threshold_dict=args.threshold_dict, ) LOG.info( @@ -172,21 +172,14 @@ def main(args: ScriptConfig): help="Stable identifier for the truth source (e.g. truth_).", ) parser.add_argument( - "--shp_regions", - type=lambda x: [r for r in x.split(",") if r], - help="Comma-separated list of shapefile paths for spatial stratification.", - default="", - ) - parser.add_argument( - "--bbox_regions", - type=lambda x: { - name: [float(v) for v in bbox.split(",")] - for part in x.split(";") - if part - for name, bbox in [part.split(":", 1)] - }, - help="Semicolon-separated bounding-box regions as name:lon_min,lon_max,lat_min,lat_max.", - default="", + "--regions", + type=json.loads, + help=( + "JSON list of region specs in config order. " + 'Each entry is {"type": "bbox", "name": ..., "bbox": [...]} ' + 'or {"type": "shp", "name": ..., "path": ...}.' + ), + default="[]", ) parser.add_argument( "--threshold_dict", diff --git a/workflow/scripts/verification_plot_metrics.py b/workflow/scripts/verification_plot_metrics.py index e2e3f02f..64d316e1 100644 --- a/workflow/scripts/verification_plot_metrics.py +++ b/workflow/scripts/verification_plot_metrics.py @@ -112,7 +112,9 @@ def main(args: Namespace) -> None: metrics = all_df["metric"].unique() params = all_df["param"].unique() seasons = all_df["season"].unique() if args.stratify else ["all"] - regions = all_df["region"].unique() if args.stratify else ["all"] + regions = ( + all_df["region"].unique() if args.stratify else [all_df["region"].unique()[0]] + ) init_hours = ( all_df["init_hour"].unique() if args.stratify else [-999] ) # numeric code to indicate all init hours diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index d5948323..1aac94ef 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -869,7 +869,7 @@ "description": "Stratification settings for the analysis.", "properties": { "regions": { - "description": "List of region specs for spatial stratification. String entries are shapefile names (resolved against 'root'). Dict entries map a region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. The special key 'all' overrides the default full-domain region; any other key adds a named bbox region.", + "description": "List of region specs for spatial stratification. At least one region is required. String entries are shapefile names (resolved against 'root'). Dict entries map a region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. The first entry is the domain region used by the dashboard when region stratification is not active (e.g. {'global': [-180, 180, -90, 90]} or {'icon-ch1': [1.5, 16, 43, 49.5]}).", "items": { "anyOf": [ { From 3e29794e688d5c09a8a72be2fd5e9f6e6309dfda Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 22 Jul 2026 17:27:14 +0200 Subject: [PATCH 20/28] Fixes --- src/verification/__init__.py | 2 -- tests/unit/test_verification.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index d4fa444b..fb301eec 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -3,8 +3,6 @@ import re import time -from pathlib import Path - import cartopy.crs as ccrs from cartopy.io.shapereader import Reader diff --git a/tests/unit/test_verification.py b/tests/unit/test_verification.py index 751f69da..96399004 100644 --- a/tests/unit/test_verification.py +++ b/tests/unit/test_verification.py @@ -44,7 +44,7 @@ def _make_verif_dataset(forecast_reference_times): }, coords={ "forecast_reference_time": forecast_reference_times, - "region": ["all"], + "region": ["global"], "source": ["fcst"], }, ) From 2a614d08905b559a080a5ea14f188e64b49d45cd Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 22 Jul 2026 17:32:15 +0200 Subject: [PATCH 21/28] Remove Michele's configs --- config/stage-a-o96-multi-step.yaml | 100 ---------------------------- config/stage-b-n320-multi-step.yaml | 95 -------------------------- 2 files changed, 195 deletions(-) delete mode 100644 config/stage-a-o96-multi-step.yaml delete mode 100644 config/stage-b-n320-multi-step.yaml diff --git a/config/stage-a-o96-multi-step.yaml b/config/stage-a-o96-multi-step.yaml deleted file mode 100644 index 4ffd5640..00000000 --- a/config/stage-a-o96-multi-step.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# yaml-language-server: $schema=../workflow/tools/config.schema.json -description: | - Evaluate skill of a stage A o96 multi-step global model against ERA. - -dates: - start: 2024-01-01T00:00 - end: 2024-07-01T00:00 - frequency: 59h - # - 2024-01-01T00:00 - -runs: - - forecaster: - checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6/a689d740f37642c38fd01a39cdbde96f/inference-last.ckpt - label: resO96_fr1_st1_in7_out6 - steps: 0/120/1 - config: resources/inference/configs/o96-global-multistep-forecaster.yaml - disable_local_eccodes_definitions: true - - forecaster: - checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6_water/b1b457100bbe42358bfab53fda8f6d67/inference-last.ckpt - label: resO96_fr1_st1_in7_out6_water - steps: 0/120/1 - config: resources/inference/configs/o96-global-multistep-forecaster.yaml - disable_local_eccodes_definitions: true - - # - forecaster: - # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in7_out6/f812c442ebee498485da0f7c4a74bb7a/inference-last.ckpt - # label: resO96_fr1_st1_in7_out6_w_1_1_2_2_4_8 - # steps: 0/120/1 - # config: resources/inference/configs/o96-global-multistep-forecaster.yaml - # disable_local_eccodes_definitions: true - - # - forecaster: - # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in2_out1/877cb9559f6c484d97b3733b7481c39a/inference-last.ckpt - # label: resO96_fr1_st1_in2_out1 - # steps: 0/120/1 - # config: resources/inference/configs/o96-global-multistep-forecaster.yaml - # disable_local_eccodes_definitions: true - - # - forecaster: - # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resO96_fr1_st1_in4_out3/70b8ec9d55144f968513b706415ffc04/inference-last.ckpt - # label: resO96_fr1_st1_in4_out3 - # steps: 0/120/1 - # config: resources/inference/configs/o96-global-multistep-forecaster.yaml - # disable_local_eccodes_definitions: true - -truth: - label: ERA5-o96 - root: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-o96-1979-2024-1h-v3-with-era51.zarr - -experiment: - params: - - T_2M - - TD_2M - - U_10M - - V_10M - - TOT_PREC - - CLCT - - CLCL - - SSRD - dashboard: - stratification: - # - region - # - init_hour - - season - stratification: - regions: [] - -showcase: - params: - - T_2M - - SP_10M - - TOT_PREC - - CLCT - meteograms: - enabled: false # Because to Jretrieve credentials - animations: - enabled: true - domains: - - name: globe - rotate: true - hours_per_revolution: 120 - # - europe - # # - alps - # - icon-ch - # - switzerland - -locations: - output_root: output/ - -profile: - executor: slurm - global_resources: - gpus: 16 - default_resources: - slurm_partition: "postproc" - cpus_per_task: 1 - mem_mb_per_cpu: 1800 - runtime: "1h" - gpus: 0 - jobs: 50 diff --git a/config/stage-b-n320-multi-step.yaml b/config/stage-b-n320-multi-step.yaml deleted file mode 100644 index d84492c5..00000000 --- a/config/stage-b-n320-multi-step.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# yaml-language-server: $schema=../workflow/tools/config.schema.json -description: | - Evaluate skill of a stage A n320 multi-step global model against ERA. - -dates: - start: 2024-01-01T00:00 - end: 2024-07-01T00:00 - frequency: 59h - # - 2024-01-01T00:00 - # - 2024-06-19T00:00 - -runs: - - forecaster: - checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in7_out6/a4bae159c2e64af6a749031737bee02e/inference-last.ckpt - label: resN320_fr1_st1_in7_out6 - steps: 0/120/1 - config: resources/inference/configs/n320-global-multistep-forecaster.yaml - disable_local_eccodes_definitions: true - - # - forecaster: - # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in7_out6_rl4/482d4dfa299143e996ec2f6962510738/inference-anemoi-by_step-epoch_005-step_012000.ckpt - # label: resN320_fr1_st1_in7_out6_rl4 - # steps: 0/120/1 - # config: resources/inference/configs/n320-global-multistep-forecaster.yaml - # disable_local_eccodes_definitions: true - - # - forecaster: - # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in2_out1/406bb2ab927e4f7eb1f5c133b53ddcbc/inference-last.ckpt - # label: resN320_fr1_st1_in2_out1 - # steps: 0/120/1 - # config: resources/inference/configs/n320-global-multistep-forecaster.yaml - # disable_local_eccodes_definitions: true - - # - forecaster: - # checkpoint: /scratch/mch/miccatta/SwissAI_checkpoints/checkpoint_resN320_fr1_st1_in4_out3/f64c33bccca74411bb267b02df8723e5/inference-last.ckpt - # label: resN320_fr1_st1_in4_out3 - # steps: 0/120/1 - # config: resources/inference/configs/n320-global-multistep-forecaster.yaml - # disable_local_eccodes_definitions: true - -truth: - label: ERA5-n320 - root: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-n320-1979-2024-1h-v2-with-era51.zarr - -experiment: - params: - - T_2M - - TD_2M - - U_10M - - V_10M - - TOT_PREC - - CLCT - dashboard: - stratification: - # - region - # - init_hour - - season - stratification: - regions: [] - -showcase: - params: - - T_2M - - SP_10M - - TOT_PREC - - CLCT - - CLCL - - SSRD - meteograms: - enabled: false # Because to Jretrieve credentials - animations: - enabled: true - domains: - - name: globe - rotate: false - hours_per_revolution: 240 # twice the horizon of 120h (it makes half a revolution) - # - europe - # # - alps - # - icon-ch - # - switzerland - -locations: - output_root: output/ - -profile: - executor: slurm - global_resources: - gpus: 16 - default_resources: - slurm_partition: "postproc" - cpus_per_task: 1 - mem_mb_per_cpu: 1800 - runtime: "1h" - gpus: 0 - jobs: 50 From 672fb7ad413c1ffcbd040c5390946497633dab36 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Wed, 22 Jul 2026 17:33:48 +0200 Subject: [PATCH 22/28] Remove leftovers following 2a614d0 --- .../n320-global-multistep-forecaster.yaml | 36 ------------------- .../o96-global-multistep-forecaster.yaml | 35 ------------------ 2 files changed, 71 deletions(-) delete mode 100644 resources/inference/configs/n320-global-multistep-forecaster.yaml delete mode 100644 resources/inference/configs/o96-global-multistep-forecaster.yaml diff --git a/resources/inference/configs/n320-global-multistep-forecaster.yaml b/resources/inference/configs/n320-global-multistep-forecaster.yaml deleted file mode 100644 index b3a076b9..00000000 --- a/resources/inference/configs/n320-global-multistep-forecaster.yaml +++ /dev/null @@ -1,36 +0,0 @@ -input: - test: - use_original_paths: true - -allow_nans: true - -patch_metadata: - config: - dataloader: - test: - datasets: - data: - dataset_config: - dataset: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-n320-1979-2024-1h-v2-with-era51.zarr - - -post_processors: - - accumulate_from_start_of_forecast: - accumulations: - - tp - - forward_transform_filter: - rescale: - scale: 1000 # convert units from m to kg m-2 - offset: 0 - param: tp - -output: - grib: - path: grib/{date}{time:04}_{step:03}.grib - # ssrd/strd have no valid value at step 0 (1h-period accumulation ending - # at the current time would need data before the forecast start) — - # "skip" omits them from the initial state instead of erroring or writing - # a physically meaningless value. - negative_step_mode: skip - -write_initial_state: true diff --git a/resources/inference/configs/o96-global-multistep-forecaster.yaml b/resources/inference/configs/o96-global-multistep-forecaster.yaml deleted file mode 100644 index ec9b9b79..00000000 --- a/resources/inference/configs/o96-global-multistep-forecaster.yaml +++ /dev/null @@ -1,35 +0,0 @@ -input: - test: - use_original_paths: true - -allow_nans: true - -patch_metadata: - config: - dataloader: - test: - datasets: - data: - dataset_config: - dataset: /store_new/mch/msopr/ml/datasets/aifs-ea-an-oper-0001-mars-o96-1979-2024-1h-v3-with-era51.zarr - -post_processors: - - accumulate_from_start_of_forecast: - accumulations: - - tp - - forward_transform_filter: - rescale: - scale: 1000 # convert units from m to kg m-2 - offset: 0 - param: tp - -output: - grib: - path: grib/{date}{time:04}_{step:03}.grib - # ssrd/strd have no valid value at step 0 (1h-period accumulation ending - # at the current time would need data before the forecast start) — - # "skip" omits them from the initial state instead of erroring or writing - # a physically meaningless value. - negative_step_mode: skip - -write_initial_state: true From 41256c72c40b8267361823d1e846292f12ba59cd Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 24 Jul 2026 09:29:33 +0200 Subject: [PATCH 23/28] Rename domain to 'icon' --- README.md | 2 +- config/forecasters-ich1-oper-fixed.yaml | 2 +- config/forecasters-ich1-oper.yaml | 2 +- config/forecasters-ich1.yaml | 2 +- config/forecasters-ich1_mec_ffv2.yaml | 2 +- config/varda-single-1.0.yaml | 2 +- src/evalml/config.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8328fd4a..21611f70 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ truth: experiment: stratification: regions: - - icon-ch1: [1.5, 16, 43, 49.5] # domain region — must be first + - icon: [1.5, 16, 43, 49.5] # domain region — must be first - jura - mittelland - voralpen diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index 6b487206..e4e57c20 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -46,7 +46,7 @@ experiment: - TOT_PREC6 stratification: regions: - - icon-ch1: [1.5, 16, 43, 49.5] + - icon: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 77f350f3..82618b9f 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -44,7 +44,7 @@ experiment: - TOT_PREC6 stratification: regions: - - icon-ch1: [1.5, 16, 43, 49.5] + - icon: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 61e164b1..282bf8a9 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -57,7 +57,7 @@ experiment: - PMSL stratification: regions: - - icon-ch1: [1.5, 16, 43, 49.5] + - icon: [1.5, 16, 43, 49.5] - jura - mittelland - voralpen diff --git a/config/forecasters-ich1_mec_ffv2.yaml b/config/forecasters-ich1_mec_ffv2.yaml index 498c6ee3..3fb59773 100644 --- a/config/forecasters-ich1_mec_ffv2.yaml +++ b/config/forecasters-ich1_mec_ffv2.yaml @@ -44,7 +44,7 @@ experiment: - TOT_PREC stratification: regions: - - icon-ch1: [1.5, 16, 43, 49.5] + - icon: [1.5, 16, 43, 49.5] - jura root: /store_new/mch/msopr/ml/regions/Prognoseregionen_LV95_20220517 thresholds: diff --git a/config/varda-single-1.0.yaml b/config/varda-single-1.0.yaml index a13f3c3a..772c74d4 100644 --- a/config/varda-single-1.0.yaml +++ b/config/varda-single-1.0.yaml @@ -54,7 +54,7 @@ experiment: - PMSL stratification: regions: - - icon-ch1: [1.5, 16, 43, 49.5] + - icon: [1.5, 16, 43, 49.5] - mittelland - berge - alpennordseite diff --git a/src/evalml/config.py b/src/evalml/config.py index 162219b5..315407c6 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -426,7 +426,7 @@ def validate_regions( raise ValueError( "At least one region must be specified. " "Add a domain bbox as the first entry, e.g. regions: [{global: [-180, 180, -90, 90]}] " - "for global models or [{icon-ch1: [1.5, 16, 43, 49.5]}] for ICON-CH1." + "for global models or [{icon: [1.5, 16, 43, 49.5]}] for ICON-CH1." ) for entry in v: if isinstance(entry, dict): From 05ba8af4ce31f611081760162fed0cfef83a5bee Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 24 Jul 2026 09:31:49 +0200 Subject: [PATCH 24/28] Update comment --- src/evalml/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/evalml/config.py b/src/evalml/config.py index 315407c6..707e12e0 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -425,8 +425,8 @@ def validate_regions( if not v: raise ValueError( "At least one region must be specified. " - "Add a domain bbox as the first entry, e.g. regions: [{global: [-180, 180, -90, 90]}] " - "for global models or [{icon: [1.5, 16, 43, 49.5]}] for ICON-CH1." + "Add a domain region as the first entry, either a bbox dict " + "(e.g. {global: [-180, 180, -90, 90]}) or a shapefile name." ) for entry in v: if isinstance(entry, dict): From 4df7af223c2ed1ae99024b14f00920b3f06b7cfa Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 24 Jul 2026 09:33:04 +0200 Subject: [PATCH 25/28] Fix docstrings --- src/verification/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/verification/__init__.py b/src/verification/__init__.py index fb301eec..9ab3777d 100644 --- a/src/verification/__init__.py +++ b/src/verification/__init__.py @@ -337,7 +337,7 @@ def verify( Label for the forecast source (used in output dataset). obs_label : str Label for the observation source (used in output dataset). - regions : list[dict] or None, optional + regions : list[dict] Ordered list of region specs. Each entry is either ``{"type": "bbox", "name": ..., "bbox": [lon_min, lon_max, lat_min, lat_max]}`` or ``{"type": "shp", "name": ..., "path": ...}``. The list order is preserved in the From e9fd7dc97b27379f34393ccda6af119f91f80a19 Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 24 Jul 2026 10:04:34 +0200 Subject: [PATCH 26/28] Use TOT_PREC6 --- config/aifs-single.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/aifs-single.yaml b/config/aifs-single.yaml index 96535b45..e72c04c6 100644 --- a/config/aifs-single.yaml +++ b/config/aifs-single.yaml @@ -34,7 +34,7 @@ experiment: - T_2M - TD_2M - SP_10M - - TOT_PREC + - TOT_PREC6 dashboard: stratification: # - region From 9b1f1f47845a4518283d17f51686d9ad1dba48bd Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 24 Jul 2026 10:59:31 +0200 Subject: [PATCH 27/28] Add predefined verif regions --- README.md | 2 +- config/aifs-single.yaml | 2 +- config/forecasters-ich1-oper-fixed.yaml | 2 +- config/forecasters-ich1-oper.yaml | 2 +- config/forecasters-ich1.yaml | 2 +- config/forecasters-ich1_mec_ffv2.yaml | 2 +- config/varda-single-1.0.yaml | 2 +- src/evalml/config.py | 17 +++++++++++------ workflow/rules/common.smk | 14 +++++++++++++- 9 files changed, 31 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 21611f70..9644065a 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ truth: experiment: stratification: regions: - - icon: [1.5, 16, 43, 49.5] # domain region — must be first + - icon # first item will be used as default verification region - jura - mittelland - voralpen diff --git a/config/aifs-single.yaml b/config/aifs-single.yaml index e72c04c6..043c5d13 100644 --- a/config/aifs-single.yaml +++ b/config/aifs-single.yaml @@ -42,7 +42,7 @@ experiment: - season stratification: regions: - - global: [-180, 180, -90, 90] + - global locations: output_root: output/ diff --git a/config/forecasters-ich1-oper-fixed.yaml b/config/forecasters-ich1-oper-fixed.yaml index e4e57c20..9409b55a 100644 --- a/config/forecasters-ich1-oper-fixed.yaml +++ b/config/forecasters-ich1-oper-fixed.yaml @@ -46,7 +46,7 @@ experiment: - TOT_PREC6 stratification: regions: - - icon: [1.5, 16, 43, 49.5] + - icon - jura - mittelland - voralpen diff --git a/config/forecasters-ich1-oper.yaml b/config/forecasters-ich1-oper.yaml index 82618b9f..d723e7f8 100644 --- a/config/forecasters-ich1-oper.yaml +++ b/config/forecasters-ich1-oper.yaml @@ -44,7 +44,7 @@ experiment: - TOT_PREC6 stratification: regions: - - icon: [1.5, 16, 43, 49.5] + - icon - jura - mittelland - voralpen diff --git a/config/forecasters-ich1.yaml b/config/forecasters-ich1.yaml index 282bf8a9..9492939b 100644 --- a/config/forecasters-ich1.yaml +++ b/config/forecasters-ich1.yaml @@ -57,7 +57,7 @@ experiment: - PMSL stratification: regions: - - icon: [1.5, 16, 43, 49.5] + - icon - jura - mittelland - voralpen diff --git a/config/forecasters-ich1_mec_ffv2.yaml b/config/forecasters-ich1_mec_ffv2.yaml index 3fb59773..8d3b4ce5 100644 --- a/config/forecasters-ich1_mec_ffv2.yaml +++ b/config/forecasters-ich1_mec_ffv2.yaml @@ -44,7 +44,7 @@ experiment: - TOT_PREC stratification: regions: - - icon: [1.5, 16, 43, 49.5] + - icon - jura root: /store_new/mch/msopr/ml/regions/Prognoseregionen_LV95_20220517 thresholds: diff --git a/config/varda-single-1.0.yaml b/config/varda-single-1.0.yaml index 772c74d4..4ee63d57 100644 --- a/config/varda-single-1.0.yaml +++ b/config/varda-single-1.0.yaml @@ -54,7 +54,7 @@ experiment: - PMSL stratification: regions: - - icon: [1.5, 16, 43, 49.5] + - icon - mittelland - berge - alpennordseite diff --git a/src/evalml/config.py b/src/evalml/config.py index 707e12e0..981a67e7 100644 --- a/src/evalml/config.py +++ b/src/evalml/config.py @@ -5,6 +5,11 @@ PROJECT_ROOT = Path(__file__).parents[2] +PREDEFINED_REGIONS: Dict[str, List[float]] = { + "global": [-180, 180, -90, 90], + "icon": [1.5, 16, 43, 49.5], +} + class Dates(BaseModel): """Start/stop of the hindcast period and the launch frequency.""" @@ -406,10 +411,10 @@ class Stratification(BaseModel): default_factory=list, description=( "List of region specs for spatial stratification. At least one region is required. " - "String entries are shapefile names (resolved against 'root'). Dict entries map a " - "region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. " - "The first entry is the domain region used by the dashboard when region stratification " - "is not active (e.g. {'global': [-180, 180, -90, 90]} or {'icon-ch1': [1.5, 16, 43, 49.5]})." + f"String entries are either predefined region names ({list(PREDEFINED_REGIONS)}) or " + "shapefile names resolved against 'root'. Predefined names take precedence over shapefiles. " + "Dict entries map a custom region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. " + "The first entry is the domain region used by the dashboard when region stratification is not active." ), ) root: Optional[str] = Field( @@ -425,8 +430,8 @@ def validate_regions( if not v: raise ValueError( "At least one region must be specified. " - "Add a domain region as the first entry, either a bbox dict " - "(e.g. {global: [-180, 180, -90, 90]}) or a shapefile name." + f"Add a domain region as the first entry, e.g. a predefined name " + f"({list(PREDEFINED_REGIONS)}), a custom bbox dict, or a shapefile name." ) for entry in v: if isinstance(entry, dict): diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 1e34f407..014d99c8 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -79,19 +79,31 @@ def parse_reference_times(): return times +PREDEFINED_REGIONS = { + "global": [-180, 180, -90, 90], + "icon": [1.5, 16, 43, 49.5], +} + + def parse_regions(): """Return a JSON list of region specs in config order. Each entry is either ``{"type": "bbox", "name": ..., "bbox": [...]}`` or ``{"type": "shp", "name": ..., "path": ...}``, preserving the original order so the NetCDF region coordinate matches the config. + + String entries are resolved as predefined region names first (see + PREDEFINED_REGIONS), falling back to shapefile lookup against 'root'. """ cfg = config["experiment"]["stratification"] root = cfg.get("root", "") result = [] for entry in cfg.get("regions", []): if isinstance(entry, str): - result.append({"type": "shp", "name": entry, "path": f"{root}/{entry}.shp"}) + if entry in PREDEFINED_REGIONS: + result.append({"type": "bbox", "name": entry, "bbox": PREDEFINED_REGIONS[entry]}) + else: + result.append({"type": "shp", "name": entry, "path": f"{root}/{entry}.shp"}) elif isinstance(entry, dict): name, bbox = next(iter(entry.items())) result.append({"type": "bbox", "name": name, "bbox": bbox}) From c9c0ac6188df83d35991c7fddf40e70116e705ab Mon Sep 17 00:00:00 2001 From: Daniele Nerini Date: Fri, 24 Jul 2026 11:03:23 +0200 Subject: [PATCH 28/28] Linting --- workflow/rules/common.smk | 14 +------------- workflow/tools/config.schema.json | 2 +- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 014d99c8..1e34f407 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -79,31 +79,19 @@ def parse_reference_times(): return times -PREDEFINED_REGIONS = { - "global": [-180, 180, -90, 90], - "icon": [1.5, 16, 43, 49.5], -} - - def parse_regions(): """Return a JSON list of region specs in config order. Each entry is either ``{"type": "bbox", "name": ..., "bbox": [...]}`` or ``{"type": "shp", "name": ..., "path": ...}``, preserving the original order so the NetCDF region coordinate matches the config. - - String entries are resolved as predefined region names first (see - PREDEFINED_REGIONS), falling back to shapefile lookup against 'root'. """ cfg = config["experiment"]["stratification"] root = cfg.get("root", "") result = [] for entry in cfg.get("regions", []): if isinstance(entry, str): - if entry in PREDEFINED_REGIONS: - result.append({"type": "bbox", "name": entry, "bbox": PREDEFINED_REGIONS[entry]}) - else: - result.append({"type": "shp", "name": entry, "path": f"{root}/{entry}.shp"}) + result.append({"type": "shp", "name": entry, "path": f"{root}/{entry}.shp"}) elif isinstance(entry, dict): name, bbox = next(iter(entry.items())) result.append({"type": "bbox", "name": name, "bbox": bbox}) diff --git a/workflow/tools/config.schema.json b/workflow/tools/config.schema.json index 1aac94ef..ea2b84ba 100644 --- a/workflow/tools/config.schema.json +++ b/workflow/tools/config.schema.json @@ -869,7 +869,7 @@ "description": "Stratification settings for the analysis.", "properties": { "regions": { - "description": "List of region specs for spatial stratification. At least one region is required. String entries are shapefile names (resolved against 'root'). Dict entries map a region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. The first entry is the domain region used by the dashboard when region stratification is not active (e.g. {'global': [-180, 180, -90, 90]} or {'icon-ch1': [1.5, 16, 43, 49.5]}).", + "description": "List of region specs for spatial stratification. At least one region is required. String entries are either predefined region names (['global', 'icon']) or shapefile names resolved against 'root'. Predefined names take precedence over shapefiles. Dict entries map a custom region name to a bounding box [lon_min, lon_max, lat_min, lat_max]. The first entry is the domain region used by the dashboard when region stratification is not active.", "items": { "anyOf": [ {