Paper figures

.gitignore

@@ -15,6 +15,7 @@ _dev
 .idea
 .snakemake
 output
+figures
 _sandbox/
 experiment_report.html
 

config/varda-single_paper_analysis.yaml

@@ -0,0 +1,135 @@
+# yaml-language-server: $schema=../workflow/tools/config.schema.json
+description: |
+  Evaluation of Varda-Single version 1.0.
+
+dates:
+  start: 2025-04-01T00:00
+  end: 2026-03-31T18:00
+  frequency: 30h
+
+runs:
+  - temporal_downscaler:
+      checkpoint: /scratch/mch/miccatta/ICON_interpolator_checkpoints/checkpoint_stage-C-interpolator-n320-6hto1h-reduced-variables/f9279244ed6f4c458597bdcf335ab36f/inference-last.ckpt
+      label: Varda-single
+      steps: 0/120/1
+      config: resources/inference/configs/sgm-temporal-downscaler-global_trimedge_multi.yaml
+      extra_requirements:
+        - anemoi-datasets==0.5.35
+        - anemoi-inference==0.11.1
+        # NOTE this pins the merge commit that introduced 'copy-prognostic-from-forecaster'
+        # TODO pin a release tag once the plugins are versioned. Do not pin the main branch to avoid stale code (changes do to main to not change the evalml hash)
+        - git+https://github.com/MeteoSwiss/anemoi-plugins-meteoswiss.git@b4835d76346e1e5ec0181884520da00e317fbfbc
+      forecaster:
+        checkpoint: https://service.meteoswiss.ch/mlstore#/experiments/602/runs/98d4e297f0164f18a0ad50fb01fbff42
+        config: resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml
+        steps: 0/120/6
+  - forecaster:
+      checkpoint: https://huggingface.co/ecmwf/aifs-single-1.1/resolve/main/aifs-single-mse-1.1.ckpt
+      label: AIFS-single-1.1
+      steps: 0/120/6
+      config: resources/inference/configs/aifs-single-forecaster.yaml
+      extra_requirements:
+        - torch-geometric==2.4.0
+        - anemoi-inference==0.6.3
+        - anemoi-models==0.5.0
+        - https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
+      disable_local_eccodes_definitions: true
+  - baseline:
+      label: ICON-CH1-CTRL
+      root: /store_new/mch/msopr/osm/ICON-CH1-EPS
+      steps: 0/33/1
+      member: control
+  - baseline:
+      label: ICON-CH2-CTRL
+      root: /store_new/mch/msopr/osm/ICON-CH2-EPS
+      steps: 0/120/1
+      member: control
+  - baseline:
+      label: ICON-CH1-EPS mean
+      root: /store_new/mch/msopr/osm/ICON-CH1-EPS
+      steps: 0/33/1
+      member: mean
+  - baseline:
+      label: ICON-CH2-EPS mean
+      root: /store_new/mch/msopr/osm/ICON-CH2-EPS
+      steps: 0/120/1
+      member: mean
+
+truth:
+  label: KENDA-CH1
+  root: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2026-1h-pl13-v2.0.zarr
+
+experiment:
+  params:
+    - T_2M
+    - TD_2M
+    - SP_10M
+    - TOT_PREC1
+    - TOT_PREC6
+    - PMSL
+  stratification:
+    regions:
+      - icon
+      - jura
+      - mittelland
+      - alpennordhang
+      - innerealpentaeler
+      - alpensuedseite
+      - alpen
+      - tiefelagen
+      - berge
+    root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517
+  thresholds:
+    TOT_PREC1:
+      gt: [0.0, 0.1, 1, 5, 10]
+    TOT_PREC6:
+      gt: [0.0, 0.1, 5, 10, 20, 50]
+    SP_10M:
+      gt: [2.5, 5.0, 10.0]
+    T_2M:
+      lt: [273.15]
+      gt: [288.15, 298.15]
+  dashboard:
+    stratification:
+      # - region
+      # - init_hour
+      - season
+  scorecards:
+    enabled: true
+    sections:
+      short_range:
+        baseline: ICON-CH1-CTRL
+        lead_times: "6/33/6"
+        stratification: region
+        variables:
+          - "U_10M:RMSE,R2,ETS"
+          - "T_2M:RMSE,R2"
+          - "TOT_PREC1:RMSE,R2,ETS"
+      medium_range:
+        baseline: ICON-CH2-CTRL
+        lead_times: "24/120/24"
+        stratification: region
+        variables:
+          - "U_10M:RMSE,R2,ETS"
+          - "T_2M:RMSE,R2"
+          - "TOT_PREC6:RMSE,R2,ETS"
+  scoremaps:
+    enabled: false
+    leadtimes: [6, 24]
+    params: ["T_2M", "SP_10M", "TOT_PREC1", "TOT_PREC6"]
+
+locations:
+  output_root: output/
+
+profile:
+  executor: slurm
+  global_resources:
+    gpus: 16
+  default_resources:
+    slurm_partition: "postproc"
+    cpus_per_task: 1
+    mem_mb_per_cpu: 1800
+    runtime: "1h"
+    gpus: 0
+    slurm_account: s83
+  jobs: 20

config/varda-single_paper_forecaster_analysis.yaml

@@ -0,0 +1,113 @@
+# yaml-language-server: $schema=../workflow/tools/config.schema.json
+description: |
+  Evaluation of Varda-Single version 1.0.
+
+dates:
+  start: 2025-04-01T00:00
+  end: 2026-03-31T18:00
+  frequency: 30h
+
+runs:
+  - forecaster:
+      label: stage-E-no-rollout
+      checkpoint: https://service.meteoswiss.ch/mlstore#/experiments/602/runs/c30490b6ba064e4db03b430f3a2595ad
+      config: resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml
+      steps: 0/120/6
+  - forecaster:
+      label: stage-E-with-rollout
+      checkpoint: https://service.meteoswiss.ch/mlstore#/experiments/602/runs/98d4e297f0164f18a0ad50fb01fbff42
+      config: resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml
+      steps: 0/120/6
+  # - forecaster:
+  #     checkpoint: https://huggingface.co/ecmwf/aifs-single-1.1/resolve/main/aifs-single-mse-1.1.ckpt
+  #     label: AIFS-single-1.1
+  #     steps: 0/120/6
+  #     config: resources/inference/configs/aifs-single-forecaster.yaml
+  #     extra_requirements:
+  #       - torch-geometric==2.4.0
+  #       - anemoi-inference==0.6.3
+  #       - anemoi-models==0.5.0
+  #       - https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl
+  #     disable_local_eccodes_definitions: true
+  - baseline:
+      label: ICON-CH1-CTRL
+      root: /store_new/mch/msopr/osm/ICON-CH1-EPS
+      steps: 0/33/6
+      member: control
+  - baseline:
+      label: ICON-CH2-CTRL
+      root: /store_new/mch/msopr/osm/ICON-CH2-EPS
+      steps: 0/120/6
+      member: control
+
+truth:
+  label: KENDA-CH1
+  root: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2026-1h-pl13-v2.0.zarr
+
+experiment:
+  params:
+    - T_2M
+    - TD_2M
+    - SP_10M
+    - TOT_PREC6
+    - PMSL
+  stratification:
+    regions:
+      - icon
+      - jura
+      - mittelland
+      - alpennordhang
+      - innerealpentaeler
+      - alpensuedseite
+      - alpen
+      - tiefelagen
+      - berge
+    root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517
+  thresholds:
+    TOT_PREC6:
+      gt: [0.0, 0.1, 5, 10, 20, 50]
+    SP_10M:
+      gt: [2.5, 5.0, 10.0]
+    T_2M:
+      lt: [273.15]
+      gt: [288.15, 298.15]
+  dashboard:
+    stratification:
+      # - region
+      # - init_hour
+      - season
+  scorecards:
+    enabled: true
+    sections:
+      short_range:
+        baseline: ICON-CH1-CTRL
+        lead_times: "6/33/6"
+        stratification: region
+        variables:
+          - "U_10M:RMSE,R2,ETS"
+          - "T_2M:RMSE,R2"
+          - "TOT_PREC6:RMSE,R2,ETS"
+      medium_range:
+        baseline: ICON-CH2-CTRL
+        lead_times: "24/120/24"
+        stratification: region
+        variables:
+          - "U_10M:RMSE,R2,ETS"
+          - "T_2M:RMSE,R2"
+          - "TOT_PREC6:RMSE,R2,ETS"
+
+locations:
+  output_root: output/
+
+profile:
+  executor: slurm
+  global_resources:
+    gpus: 16
+  default_resources:
+    slurm_partition: "postproc"
+    cpus_per_task: 1
+    mem_mb_per_cpu: 1800
+    runtime: "1h"
+    gpus: 0
+    slurm_account: s83
+  jobs: 20

config/varda-single_paper_forecaster_scoremaps.yaml

@@ -0,0 +1,90 @@
+# yaml-language-server: $schema=../workflow/tools/config.schema.json
+description: |
+  Evaluation of Varda-Single version 1.0.
+
+dates:
+  start: 2025-04-01T00:00
+  end: 2026-03-31T18:00
+  frequency: 30h
+
+runs:
+  - forecaster:
+      label: Varda-single
+      checkpoint: https://service.meteoswiss.ch/mlstore#/experiments/602/runs/98d4e297f0164f18a0ad50fb01fbff42
+      config: resources/inference/configs/sgm-multidataset-forecaster-global-ich1-oper.yaml
+      steps: 0/120/6
+  - baseline:
+      label: ICON-CH1-CTRL
+      root: /store_new/mch/msopr/osm/ICON-CH1-EPS
+      steps: 0/33/6
+      member: control
+
+truth:
+  label: KENDA-CH1
+  root: /store_new/mch/msopr/ml/datasets/mch-ich1-1km-2024-2026-1h-pl13-v2.0.zarr
+
+experiment:
+  params:
+    - T_2M
+    - TD_2M
+    - SP_10M
+    - TOT_PREC6
+    - PMSL
+  stratification:
+    regions:
+      - icon
+      - jura
+      - mittelland
+      - alpennordhang
+      - innerealpentaeler
+      - alpensuedseite
+      - alpen
+      - tiefelagen
+      - berge
+    root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517
+  thresholds:
+    TOT_PREC6:
+      gt: [0.0, 0.1, 5, 10, 20, 50]
+    SP_10M:
+      gt: [2.5, 5.0, 10.0]
+    T_2M:
+      lt: [273.15]
+      gt: [288.15, 298.15]
+  dashboard:
+    stratification:
+      # - region
+      # - init_hour
+      - season
+  scorecards:
+    enabled: false
+    sections:
+      short_range:
+        baseline: ICON-CH1-CTRL
+        lead_times: "6/33/6"
+        stratification: region
+        variables:
+          - "U_10M:RMSE,R2,ETS"
+          - "T_2M:RMSE,R2"
+          - "TOT_PREC6:RMSE,R2,ETS"
+  scoremaps:
+    enabled: true
+    leadtimes: [6, 24]
+    params: ["T_2M", "SP_10M", "TOT_PREC6", "PMSL"]
+    scores: ["BIAS", "RMSE"]
+
+
+locations:
+  output_root: output/
+
+profile:
+  executor: slurm
+  global_resources:
+    gpus: 16
+  default_resources:
+    slurm_partition: "postproc"
+    cpus_per_task: 1
+    mem_mb_per_cpu: 1800
+    runtime: "1h"
+    gpus: 0
+    slurm_account: s83
+  jobs: 20