DT-459: extract sites config

.gitignore

@@ -55,3 +55,6 @@ __pycache__
 
 # Spack build cache
 build-linux-*
+
+# Jetbrains IDEs
+.idea/

.mch-ci.yml

@@ -70,13 +70,10 @@ default:
               pullAlways: false
               removeLocalImage: false
               pythonSourcePackage: utils
-          - pythonContainerTypeCheck:
-              imageName: ${var.containerImageName}-tester
-              imageTag: ${var.semanticVersion}
-              packageManager: ''
-              pullAlways: false
-              removeLocalImage: false
-              pythonSourcePackage: utils
+          - script: |
+              podman run --rm -w /src/app-root/utils -v $PWD:/src/app-root:rw \
+                  ${var.containerImageName}-tester:${var.semanticVersion} \
+                  sh -c "mypy -p flexpart_ifs_utils | grep error | tee /src/app-root/test_reports/mypy.log"
   - publish:
       - testArtifacts:
           - containerPublishImage:

Dockerfile

@@ -22,6 +22,7 @@ WORKDIR /opt
 # structure otherwise when it forms the .ctx directory
 COPY options /opt/options
 COPY options.meteoswiss /opt/options.meteoswiss
+COPY templates /opt/templates
 COPY spack_env /opt/spack_env
 COPY spack_repo /opt/spack_repo
 COPY src /opt/src

entrypoint.sh

@@ -6,7 +6,7 @@
 # This involves configurnig the input namelists (such as COMMAND, AVAILABLE, RELEASES, OUTGRID) based on a set of environment variables,
 # symlinking the data into the job folder, and writing the job script with the relevent paths to the input files.
 #
-# Then the job files for each release site in runtime_configuration.yaml are run - this runs Flexpart.
+# Then the job for RELEASE_SITE_NAME is run - this runs Flexpart.
 #
 # Finally the flexpart_ifs_utils library is called to upload the output of Flexpart to an S3 bucket.
 
@@ -26,15 +26,10 @@ python -m flexpart_ifs_utils generate \
 
 echo JOBS_DIR: $JOBS_DIR
 
-# Extract names from generated YAML config
-names=$(grep -oP 'name: \K.*' $JOBS_DIR/runtime_configuration.yaml)
-
-for name in $names; do
-    echo Running Flexpart IFS for release site: $name
-    cd $JOBS_DIR/$name
-    # Run Flexpart-IFS
-    bash job
-done
+echo Running Flexpart IFS for release site: $RELEASE_SITE_NAME
+cd $JOBS_DIR/$RELEASE_SITE_NAME
+# Run Flexpart-IFS
+bash job
 
 cd $SCRIPT_DIR
 # Upload output files of Flexpart-IFS to S3 bucket.

options.meteoswiss/COMMAND

@@ -33,5 +33,4 @@
  LINIT_COND=            0, ! Output sensitivity to initial conditions (bkw mode only) [0]off 1]conc 2]mmr
  SURF_ONLY=             0, ! Output only for the lowest model layer, used w/ LINIT_COND=1 or 2
  CBLFLAG=               0, ! Skewed, not Gaussian turbulence in the convective ABL, need large CTL and IFINE
- OHFIELDS_PATH= "../../flexin/", ! Default path for OH file
 /

options/COMMAND

@@ -33,5 +33,4 @@
  LINIT_COND=            0, ! Output sensitivity to initial conditions (bkw mode only) [0]off 1]conc 2]mmr 
  SURF_ONLY=             0, ! Output only for the lowest model layer, used w/ LINIT_COND=1 or 2
  CBLFLAG=               0, ! Skewed, not Gaussian turbulence in the convective ABL, need large CTL and IFINE
- OHFIELDS_PATH= "../../flexin/", ! Default path for OH file
  /

spack_repo/flexpart/packages/flexpart_ifs/package.py

@@ -47,3 +47,6 @@ def install(self, spec, prefix):
         install_tree('options', join_path(prefix.share, 'options'))
         install_tree('options.meteoswiss',
                      join_path(prefix.share, 'options.meteoswiss'))
+        # Anonymized COMMAND/RELEASES templates; flexpart_ifs_utils renders these per job instead of
+        # patching the skeletons in the options directories above.
+        install_tree('templates', join_path(prefix.share, 'templates'))

templates/COMMAND.j2

@@ -0,0 +1,42 @@
+!******************************************************************************
+!*                                                                            *
+!*      Input file for the Lagrangian particle dispersion model FLEXPART      *
+!*                         Please select your options                         *
+!*                                                                            *
+!******************************************************************************
+!
+! Anonymized control-file template. Lines a job cannot vary are literals; the templated values come
+! from the per-site config published to S3 by Terraform (`site`) and from the run's resolved time
+! window (`job`, derived from FORECAST_DATETIME plus the site's offsets). Adding a knob means
+! templating a line here - the site config deliberately carries no namelist key names and no dates.
+!
+&COMMAND
+ LDIRECT={{ site.direction_value }}, ! Simulation direction in time   ; 1 (forward) or -1 (backward)
+ IBDATE={{ job.simulation_start | fp_date }}, ! Start date of the simulation   ; YYYYMMDD: YYYY=year, MM=month, DD=day  
+ IBTIME={{ job.simulation_start | fp_time }}, ! Start time of the simulation   ; HHMISS: HH=hours, MI=min, SS=sec; UTC
+ IEDATE={{ job.simulation_end | fp_date }}, ! End date of the simulation     ; same format as IBDATE 
+ IETIME={{ job.simulation_end | fp_time }}, ! End  time of the simulation    ; same format as IBTIME
+ LOUTSTEP={{ site.output_interval_s }}, ! Interval of model output; average concentrations calculated every LOUTSTEP (s)  
+ LOUTAVER=          10800, ! Interval of output averaging (s)
+ LOUTSAMPLE=          900, ! Interval of output sampling  (s), higher stat. accuracy with shorter intervals
+ ITSPLIT=       999999999, ! Interval of particle splitting (s)
+ LSYNCTIME=           900, ! All processes are synchronized to this time interval (s)
+ CTL=                -5.0, ! CTL>1, ABL time step = (Lagrangian timescale (TL))/CTL, uses LSYNCTIME if CTL<0
+ IFINE=                 4, ! Reduction for time step in vertical transport, used only if CTL>1
+ IOUT=                  9, ! Output type: [1]mass 2]pptv 3]1&2 4]plume 5]1&4, +8 for NetCDF output
+ IPOUT=                 0, ! Particle position output: 0]no 1]every output 2]only at end 3]time averaged
+ LSUBGRID=              0, ! Increase of ABL heights due to sub-grid scale orographic variations;[0]off 1]on 
+ LCONVECTION=           1, ! Switch for convection parameterization;0]off [1]on
+ LAGESPECTRA=           0, ! Switch for calculation of age spectra (needs AGECLASSES);[0]off 1]on
+ IPIN=                  0, ! Warm start from particle dump (needs previous partposit_end file); [0]no 1]yes  
+ IOUTPUTFOREACHRELEASE= 1, ! Separate output fields for each location in the RELEASE file; [0]no 1]yes 
+ IFLUX=                 0, ! Output of mass fluxes through output grid box boundaries
+ MDOMAINFILL=           0, ! Switch for domain-filling, if limited-area particles generated at boundary
+ IND_SOURCE=            1, ! Unit to be used at the source   ;  [1]mass 2]mass mixing ratio 
+ IND_RECEPTOR=          1, ! Unit to be used at the receptor; [1]mass 2]mass mixing ratio 3]wet depo. 4]dry depo.
+ MQUASILAG=             0, ! Quasi-Lagrangian mode to track individual numbered particles 
+ NESTED_OUTPUT=         0, ! Output also for a nested domain
+ LINIT_COND=            0, ! Output sensitivity to initial conditions (bkw mode only) [0]off 1]conc 2]mmr
+ SURF_ONLY=             0, ! Output only for the lowest model layer, used w/ LINIT_COND=1 or 2
+ CBLFLAG=               0, ! Skewed, not Gaussian turbulence in the convective ABL, need large CTL and IFINE
+/

templates/RELEASES.j2

@@ -0,0 +1,32 @@
+!******************************************************************************
+!*                                                                            *
+!*      Input file for the Lagrangian particle dispersion model FLEXPART      *
+!*                         Please select your options                         *
+!*                                                                            *
+!******************************************************************************
+!
+! Anonymized control-file template. Lines a job cannot vary are literals; the templated values come
+! from the per-site config published to S3 by Terraform (`site`) and from the run's resolved time
+! window (`job`, derived from FORECAST_DATETIME plus the site's offsets). Adding a knob means
+! templating a line here - the site config deliberately carries no namelist key names and no dates.
+!
+&RELEASES_CTRL
+ NSPEC={{ site.nspec }}, ! Total number of species
+ SPECNUM_REL={{ site.species }}, ! Species numbers in directory SPECIES
+/
+&RELEASE                   ! For each release
+ IDATE1={{ job.release_start | fp_date }}, ! Release start date, YYYYMMDD: YYYY=year, MM=month, DD=day
+ ITIME1={{ job.release_start | fp_time }}, ! Release start time in UTC HHMISS: HH hours, MI=minutes, SS=seconds
+ IDATE2={{ job.release_end | fp_date }}, ! Release end date, same as IDATE1
+ ITIME2={{ job.release_end | fp_time }}, ! Release end time, same as ITIME1
+ LON1={{ site.longitude }}, ! Left longitude of release box -180 < LON1 <180
+ LON2={{ site.longitude }}, ! Right longitude of release box, same as LON1
+ LAT1={{ site.latitude }}, ! Lower latitude of release box, -90 < LAT1 < 90
+ LAT2={{ site.latitude }}, ! Upper latitude of release box same format as LAT1 
+ Z1={{ site.height_m }}, ! Lower height of release box meters/hPa above reference level
+ Z2={{ site.height_m }}, ! Upper height of release box meters/hPa above reference level
+ ZKIND={{ site.zkind }}, ! Reference level 1=above ground, 2=above sea level, 3 for pressure in hPa
+ MASS={{ site.mass_bq | fortran_real }}, ! Total mass emitted, only relevant for fwd simulations
+ PARTS   =         200000, ! Total number of particles to be released
+ COMMENT="{{ site.comment }}", ! Comment, written in the outputfile
+/

utils/flexpart_ifs_utils/__main__.py

@@ -16,58 +16,29 @@
     python __main__.py generate
         -f <flexpart_dir>
         -j <jobs_dir>
-        --datetime <YYYYMMDDHH>
-        --site BEZ
+        --datetime <YYYYMMDDhhmm>
+        --site <site>
 
     python __main__.py upload -d <jobs_dir> -i <input_directory>
 """
 
 import argparse
+import json
 import logging
 import os
 import sys
 from pathlib import Path
 
-import yaml
-
 from flexpart_ifs_utils import CONFIG
-from flexpart_ifs_utils.model import EnvironmentParameters, Model
+from flexpart_ifs_utils.job_config import resolve_job_config
+from flexpart_ifs_utils.model import Model
 from flexpart_ifs_utils.prepare_flexpart import (_path_list,
                                                  prepare_job_directory,
-                                                 render_template, select_files)
-from flexpart_ifs_utils.s3_utils import (download_keys_from_bucket,
+                                                 select_files)
+from flexpart_ifs_utils.s3_utils import (canonicalize_output_names,
+                                         download_keys_from_bucket,
                                          upload_output)
-
-
-def validate_env(data: dict[str, str | None]) -> None:
-    violations: list[str] = []
-    for parameter in EnvironmentParameters:
-        if parameter.name not in data:
-            violations.append(parameter.name)
-        elif data[parameter.name] is None:
-            violations.append(parameter.name)
-
-    if violations:
-        raise RuntimeError(
-            "Environment is missing variables needed to prepare runtime configuration: "
-            f"{violations}"
-        )
-
-
-def parse_env() -> dict[str, str | None]:
-    return {"EMISSION_START_YYYY": os.getenv("EMISSION_START_YYYY"),
-            "EMISSION_START_MM": os.getenv("EMISSION_START_MM"),
-            "EMISSION_START_DD": os.getenv("EMISSION_START_DD"),
-            "EMISSION_START_ZZ": os.getenv("EMISSION_START_ZZ"),
-            "EMISSION_END_YYYY": os.getenv("EMISSION_END_YYYY"),
-            "EMISSION_END_MM": os.getenv("EMISSION_END_MM"),
-            "EMISSION_END_DD": os.getenv("EMISSION_END_DD"),
-            "EMISSION_END_ZZ": os.getenv("EMISSION_END_ZZ"),
-            "SIMULATION_END_YYYY": os.getenv("SIMULATION_END_YYYY"),
-            "SIMULATION_END_MM": os.getenv("SIMULATION_END_MM"),
-            "SIMULATION_END_DD": os.getenv("SIMULATION_END_DD"),
-            "SIMULATION_END_ZZ": os.getenv("SIMULATION_END_ZZ")}
-
+from flexpart_ifs_utils.site_config import load_site_config
 
 if __name__ == '__main__':
 
@@ -87,7 +58,7 @@ def parse_env() -> dict[str, str | None]:
                     required=True
                     )
     p1.add_argument('--datetime',
-                    help='Forecast datetime, in format YYYYMMDDHH.',
+                    help='Forecast reference datetime, in format YYYYMMDDhhmm.',
                     required=True
                     )
 
@@ -103,7 +74,7 @@ def parse_env() -> dict[str, str | None]:
                     type=Path,
                     )
     p2.add_argument('--datetime',
-                    help='Forecast datetime, in format YYYYMMDDHH.',
+                    help='Forecast reference datetime, in format YYYYMMDDhhmm.',
                     required=True
                     )
     p2.add_argument('--site',
@@ -119,6 +90,10 @@ def parse_env() -> dict[str, str | None]:
     args = parser.parse_args()
 
     if "directory" in args:
+        # Rename before upload, so the object key never carries Flexpart's simulation-start stamp -
+        # that stamp is what forced the render step and the ensemble aggregator to re-derive the
+        # release offset, and it is why the offset could not vary per site.
+        canonicalize_output_names(args.directory / args.site / 'output')
         upload_output(args.directory, args.site, args.datetime, parent='output')
         sys.exit(0)
 
@@ -128,32 +103,27 @@ def parse_env() -> dict[str, str | None]:
     FLEXPART_DIR: Path = args.flexpart_dir
     MODEL: Model = Model(args.model)
 
-    WORKDIR: Path = Path(os.path.abspath(__file__)).parent
-    CONFIG_TEMPLATE_PATH = WORKDIR / 'runtime_configuration.j2'
-    CONFIG_PATH =  JOBS_DIR / (CONFIG_TEMPLATE_PATH.stem + '.yaml')
-
     if not os.path.exists( JOBS_DIR ):
         os.makedirs( JOBS_DIR )
 
     _logger.info('FLEXPART directory: %s', FLEXPART_DIR)
     _logger.info('Jobs directory: %s', JOBS_DIR)
     _logger.debug('Args: %s', args)
 
-    environment = parse_env()
-
-    validate_env(environment)
-
-    render_template(CONFIG_TEMPLATE_PATH, CONFIG_PATH, [RELEASE_SITE], environment)
-
-    with open(CONFIG_PATH, 'r', encoding="utf-8") as f:
-        configs = yaml.safe_load(f)
-
-    configs = [config for config in configs if config['name'] == RELEASE_SITE]
-    if not configs:
-        raise RuntimeError(f'Release site {RELEASE_SITE} does not match any known to Flexpart.')
-    if len(configs) > 1:
-        raise RuntimeError(f'Release site {RELEASE_SITE} matches multiple configs.')
-    config = configs[0]
+    # The site catalog is owned by dispersionmodelling-deployment and published to S3 by
+    # Terraform, one object per site actually configured for this environment - an unknown
+    # RELEASE_SITE_NAME fails here (S3 404) rather than matching against a locally-packaged
+    # catalog that could silently be stale relative to Terraform's config.
+    site_config_key = f'{CONFIG.main.runtime_config.site_config_key_prefix}{RELEASE_SITE}.yaml'
+    download_keys_from_bucket([site_config_key], JOBS_DIR, CONFIG.main.aws.s3.site_config)
+
+    # Plain declarative site data - no longer a Jinja template of the namelist, so there is nothing
+    # to render here and no intermediate file. The namelist templates live in the image.
+    site = load_site_config(JOBS_DIR / f'{RELEASE_SITE}.yaml')
+    # On-demand runs may override the site's own offsets for this one job; scheduled runs leave
+    # this empty, in which case the site's config applies. See resolve_job_config.
+    overrides = json.loads(os.getenv('JOB_OVERRIDES', '{}'))
+    job = resolve_job_config(FORECAST_DATETIME, MODEL, site, overrides)
 
     DATA_DIR = JOBS_DIR / 'data'
     if not os.path.exists( DATA_DIR ):
@@ -164,15 +134,15 @@ def parse_env() -> dict[str, str | None]:
 
     if not data_paths:
         # Search the db for the relevant files and download data
-        keys = select_files(config['command'],
-                            forecast_datetime=FORECAST_DATETIME,
+        keys = select_files(job,
                             step_unit=CONFIG.main.input.step_unit,
                             model=MODEL)
 
         download_keys_from_bucket(keys, DATA_DIR, CONFIG.main.aws.s3.nwp_model_data)
 
     job_dir = prepare_job_directory(
-        config,
+        site,
+        job,
         JOBS_DIR,
         FLEXPART_DIR,
         DATA_DIR,

utils/flexpart_ifs_utils/config/service_settings.py

@@ -11,6 +11,7 @@ class Bucket(BaseModel):
 class S3(BaseModel):
     nwp_model_data: Bucket
     output: Bucket
+    site_config: Bucket
 
 class AWS(BaseModel):
     s3: S3
@@ -22,11 +23,15 @@ class OpenMPConfig(BaseModel):
     num_threads: int
     stack_size: str
 
+class RuntimeConfigSettings(BaseModel):
+    site_config_key_prefix: str
+
 class AppSettings(BaseModel):
     app_name: str
     aws: AWS
     input: InputSettings
     openmp_config: OpenMPConfig
+    runtime_config: RuntimeConfigSettings
 
 class ServiceSettings(BaseServiceSettings):
     logging: LoggingSettings

utils/flexpart_ifs_utils/config/settings.yaml

@@ -13,6 +13,8 @@ main:
     stack_size: 100M
   input:
     step_unit: hours
+  runtime_config:
+    site_config_key_prefix: "sites/"
   aws:
     s3:
       nwp_model_data:
@@ -23,3 +25,7 @@ main:
         region: eu-central-2
         name: flexpart-output
         retries: 10
+      site_config:
+        region: eu-central-2
+        name: flexpart-ifs-static-input
+        retries: 10