From dbdcfa2df1a64da5293948939ddfbe592ec799bd Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Tue, 16 Jun 2026 13:43:30 +1000 Subject: [PATCH 1/8] Trying MPR --- .../app/metplus_point_stat/file/nci/pointstat.conf | 5 +---- .../app/metplus_point_stat/file/nci/surface.conf | 5 +++-- .../cset_workflow/app/verpy_metloader/file/metloader.conf | 2 +- src/CSET/cset_workflow/includes/metplus_point_stat.cylc | 4 ++-- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf index 5d7b1880d..a3d6ed571 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf @@ -36,10 +36,7 @@ INIT_TIME_FMT = %Y%m%dT%H INIT_BEG = {ENV[TASK_START_TIME]} INIT_END = {ENV[TASK_START_TIME]} INIT_INCREMENT = 1H -LEAD_SEQ = begin_end_incr(0,{ENV[FORECAST_LENGTH]},1) - -# Number of seconds to shift times in the fcst file (try half the time increment) -FCST_SHIFT = 1800 +LEAD_SEQ = begin_end_incr(4,{ENV[FORECAST_LENGTH]},1) ### # File I/O diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf index d61168298..c518fca8d 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf @@ -17,7 +17,7 @@ POINT_STAT_FCST_FILE_TYPE = NETCDF_NCCF POINT_STAT_MESSAGE_TYPE = "ADPSFC" FCST_POINT_STAT_VAR1_NAME = air_temperature -FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR1_THRESH = <=273, >273 OBS_POINT_STAT_VAR1_NAME = t2m @@ -25,7 +25,7 @@ OBS_POINT_STAT_VAR1_LEVELS = Z0 OBS_POINT_STAT_VAR1_THRESH = <=273, >273 FCST_POINT_STAT_VAR2_NAME = relative_humidity -FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR2_THRESH = <60, >95 OBS_POINT_STAT_VAR2_NAME = rh2m @@ -36,3 +36,4 @@ OBS_POINT_STAT_WINDOW_BEGIN = -1800 OBS_POINT_STAT_WINDOW_END = 1800 POINT_STAT_OUTPUT_FLAG_CNT = BOTH +POINT_STAT_OUTPUT_FLAG_MPR = BOTH diff --git a/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf b/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf index 269fb051c..0eb8ed9a6 100644 --- a/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf +++ b/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf @@ -17,7 +17,7 @@ max_fcrs = 240 # Elements used to pattern match files # MET output types as a comma separated list -stat_type = cnt +stat_type = mpr # File prefix, often the name of the MET tool, defaults to "" stream = point_stat # Optional grid filter, defaults to "" diff --git a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc index 894c04cf0..e93499032 100644 --- a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc +++ b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc @@ -63,14 +63,14 @@ [[[environment]]] ROSE_TASK_APP = metplus_point_stat STAT = area - STAT_TYPE_LIST = cnt + STAT_TYPE_LIST = cnt mpr [[METPLUS_POINT_STAT_POSTPROC]] # Family grouping all metplus Post-processing inherit = METPLUS [[[environment]]] STAT = area - STAT_TYPE_LIST = cnt + STAT_TYPE_LIST = cnt mpr [[VERPY_METLOADER]] # Family grouping for all VerPy metloader tasks From 3e7c4fa5bac3c5ac472b38687ddb6fe845348470 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:44:41 +1000 Subject: [PATCH 2/8] Update NCI site to allow changing module and CSET path --- .../opt/rose-suite-nci-gadi.conf | 14 +++++++++++ src/CSET/cset_workflow/site/nci-gadi.cylc | 25 ++++++++++++------- 2 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf diff --git a/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf new file mode 100644 index 000000000..30495ae90 --- /dev/null +++ b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf @@ -0,0 +1,14 @@ +[template variables] +SITE="nci-gadi" + +# Storage flags to add +NCI_STORAGE=["gdata/dp9"] + +# Where to store output website (view from ARE virtual desktop) +WEB_DIR="~/public_html/$CYLC_WORKFLOW_NAME" + +# Default module +CSET_ENV_USE_MODULES = True +CSET_ENV_SEPARATE_MET = False +MODULES_PURGE = "" +MODULES_LIST = "/g/data/access/ngm/modules/cset/26.4.0" diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index 543ef4ea5..d2171247e 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -2,10 +2,14 @@ # Site configuration for NCI Gadi # +# Using the opt file 'nci-gadi' is recommended to ensure the CSET module is +# loaded correctly +# # Important rose-suite.conf settings: # SITE = "nci-gadi" # Enable the site # PROJECT # NCI project code # VERPY_DIR # Path to Verpy source code +# NCI_STORAGE # List of NCI storage locations to use # METPLUS_OBS_SYSTEM # Where to source obs from # # Can be a Bureau forecast system e.g. # # 'access_c3_dn' or 'custom' @@ -14,16 +18,19 @@ # # Leave METPLUS_FCST_DIR and METPLUS_OBS_DIR unset +{% set STORAGE = [ + 'scratch/'~PROJECT, + 'gdata/'~PROJECT, + 'gdata/access', + 'gdata/hr22', +] + (NCI_STORAGE | default([])) %} + [runtime] [[root]] platform = gadi_background - init-script = """ - module use /g/data/access/ngm/modules - module load cset/26.4.0 - """ [[[ environment ]]] PROJECT = {{ PROJECT }} - PYTHONPATH = "{{VERPY_DIR}}" + PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}{% endif %} {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %} [[METPLUS]] @@ -34,14 +41,14 @@ -q = normal -l ncpus = 1 -l mem = 4gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/ig2+scratch/dx2 + -l storage = {{ STORAGE | join('+') }}+gdata/ig2 -W umask = 0022 [[[ environment ]]] ANALYSIS_LENGTH = {{ANALYSIS_LENGTH}} [[metplus_prep_obs]] [[[ environment ]]] - OBS_TIMES = R{{ANALYSIS_LENGTH | duration_as('h') | int }}/$CYLC_TASK_CYCLE_POINT/PT1H + OBS_TIMES = R{{(ANALYSIS_LENGTH | duration_as('h') | int) + 1}}/$CYLC_TASK_CYCLE_POINT/PT1H {% if METPLUS_OBS_SYSTEM != "custom" %} # Forecast system to pull obs from (e.g. 'access_c3_dn') METPLUS_OBS_SYSTEM = {{METPLUS_OBS_SYSTEM}} @@ -60,7 +67,7 @@ -q = normal -l ncpus = 48 -l mem = 180gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2 + -l storage = {{ STORAGE | join('+') }} -W umask = 0022 [[[ environment ]]] BUNCH_POOL_SIZE = $PBS_NCPUS @@ -73,7 +80,7 @@ -q = normal -l ncpus = 48 -l mem = 180gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2 + -l storage = {{ STORAGE | join('+') }} -W umask = 0022 [[[ environment ]]] BUNCH_POOL_SIZE = $PBS_NCPUS From fb566809a48b2ea6355dab1ef539674bef9ca7f1 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:46:38 +1000 Subject: [PATCH 3/8] Remove time shift now that times are correctly processed --- .../app/metplus_point_stat/file/nci/pointstat.conf | 3 --- .../app/metplus_point_stat/file/nci/surface.conf | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf index 5d7b1880d..52bf7c95b 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf @@ -38,9 +38,6 @@ INIT_END = {ENV[TASK_START_TIME]} INIT_INCREMENT = 1H LEAD_SEQ = begin_end_incr(0,{ENV[FORECAST_LENGTH]},1) -# Number of seconds to shift times in the fcst file (try half the time increment) -FCST_SHIFT = 1800 - ### # File I/O # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf index d61168298..e1c42c5b4 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf @@ -17,7 +17,7 @@ POINT_STAT_FCST_FILE_TYPE = NETCDF_NCCF POINT_STAT_MESSAGE_TYPE = "ADPSFC" FCST_POINT_STAT_VAR1_NAME = air_temperature -FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR1_THRESH = <=273, >273 OBS_POINT_STAT_VAR1_NAME = t2m @@ -25,7 +25,7 @@ OBS_POINT_STAT_VAR1_LEVELS = Z0 OBS_POINT_STAT_VAR1_THRESH = <=273, >273 FCST_POINT_STAT_VAR2_NAME = relative_humidity -FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR2_THRESH = <60, >95 OBS_POINT_STAT_VAR2_NAME = rh2m From 8545fe17f1f2200fa3b20e4796619690aa4eccab Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:47:50 +1000 Subject: [PATCH 4/8] Change ODB2 processing to output one ASCII file per valid time --- .../metplus_ascii2nc/file/nci/ascii2nc.conf | 10 +------- .../app/metplus_prep_obs/bin/odb2/odb2.py | 25 +++++++++++++++---- .../app/metplus_prep_obs/bin/prepBureauNCI.py | 13 +++------- .../app/metplus_prep_obs/bin/prepODB2.py | 16 ++++-------- .../opt/rose-app-nci-gadi.conf | 2 +- .../metplus_prep_obs/opt/rose-app-odb2.conf | 2 +- 6 files changed, 31 insertions(+), 37 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf index f0f5dddf4..ee77d8c68 100644 --- a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf +++ b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf @@ -43,16 +43,8 @@ LEAD_SEQ = # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info ### ASCII2NC_INPUT_DIR = {INPUT_BASE} -ASCII2NC_INPUT_TEMPLATE = *.ascii +ASCII2NC_INPUT_TEMPLATE = obs.{valid?fmt=%Y%m%dT%H%MZ}.ascii ASCII2NC_INPUT_FORMAT = met_point ASCII2NC_OUTPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/obs_nc ASCII2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%dT%H}.nc - -### -# ASCII2NC Settings -# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc -### - -ASCII2NC_WINDOW_BEGIN = 0 -ASCII2NC_WINDOW_END = 0 diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py index ad7f45110..2f7110f93 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py @@ -11,6 +11,8 @@ from glob import glob from pathlib import Path from typing import Iterable, TextIO +import sys +from contextlib import nullcontext import metomi.isodatetime.parsers import numpy @@ -340,13 +342,26 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: """Read ODB2 data.""" raise NotImplementedError - def odb2ascii(self, output: TextIO, valid_times: Iterable[TimePoint]): - """Write all the observations to a MET ASCII file.""" + def odb2ascii(self, output_pattern: str, valid_times: Iterable[TimePoint]): + """ + Write all the observations to a MET ASCII file. + + If output_pattern contains a strftime-style pattern then the valid time + will be used to replace the pattern. + """ for t in valid_times: + output = t.strftime(output_pattern) + + if output == '-': + out_context = nullcontext(sys.stdout) + else: + out_context = open(output, "wt") + log.info("Processing %s", t) - for obs in self.read_odb(t): - ascii = odb2ascii_dataframe(obs) - write_ascii(ascii, output) + with out_context as f: + for obs in self.read_odb(t): + ascii = odb2ascii_dataframe(obs) + write_ascii(ascii, f) class PrepODB2Pattern(PrepODB2): diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py index 2c3e1a191..90e993814 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py @@ -6,17 +6,16 @@ Valid times can be either ISO timepoints or recurrences, and are used to replace any strftime patterns. Data is sourced from the mirror in the ig2 project, not all times are available. - ./prepODB2.py \ + ./prepBureauNCI.py \ --system access-c3-dn \ --valid-time 20010101T0000Z \ --valid-time R4/20010102T0000Z/PT6H \ - --output obs.ascii + --output obs.%Y%m%dT%H%MZ.ascii """ import argparse import logging import sys -from contextlib import nullcontext from odb2 import valid_times_iterator from odb2.bom import BOM_SYSTEMS, PrepBomNci @@ -56,13 +55,7 @@ def main(argv: list[str]): logging.basicConfig(level=logging.INFO) sys.tracebacklimit = 0 - if args.output == "-": - out_context = nullcontext(sys.stdout) - else: - out_context = open(args.output, "wt") - - with out_context as output: - PrepBomNci(args.system).odb2ascii(output, valid_times_iterator(args.valid_time)) + PrepBomNci(args.system).odb2ascii(args.output, valid_times_iterator(args.valid_time)) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py index c021a261d..97a5103bd 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py @@ -9,7 +9,7 @@ ./prepODB2.py /path/to/%Y/%m/%Y%m%dT%H%MZ/*.odb \ --valid-time 20010101T0000Z \ --valid-time R4/20010102T0000Z/PT6H \ - --output obs.ascii + --output obs.%Y%m%dT%H%MZ.ascii """ import argparse @@ -71,16 +71,10 @@ def main(argv: list[str]): # Valid time unset, hopefully the pattern isn't using times args.valid_time = ["00010101T0000Z"] - if args.output == "-": - out_context = nullcontext(sys.stdout) - else: - out_context = open(args.output, "wt") - - with out_context as output: - for pattern in args.file: - PrepODB2Pattern(pattern).odb2ascii( - output, valid_times_iterator(args.valid_time) - ) + for pattern in args.file: + PrepODB2Pattern(pattern).odb2ascii( + args.output, valid_times_iterator(args.valid_time) + ) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf index 61375dd49..1fa93ef26 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf @@ -1,4 +1,4 @@ [command] # Read in files from ig2 default = mkdir -p "$METPLUS_OBS_DIR" - = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" + = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf index a47fea382..7ad4f9f67 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf @@ -1,4 +1,4 @@ [command] # Read in files from ODB2 files by providing a strftime pattern default = mkdir -p "$METPLUS_OBS_DIR" - = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" $CUSTOM_ODB2_PATTERN + = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" $CUSTOM_ODB2_PATTERN From 6c8a693133294c66695d278dacb181e040d9c70d Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 09:48:25 +1000 Subject: [PATCH 5/8] Lint --- .../cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py | 8 ++++---- .../app/metplus_prep_obs/bin/prepBureauNCI.py | 4 +++- .../cset_workflow/app/metplus_prep_obs/bin/prepODB2.py | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py index 2f7110f93..c4446a589 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py @@ -7,12 +7,12 @@ import functools import json import logging +import sys from abc import ABC, abstractmethod +from contextlib import nullcontext from glob import glob from pathlib import Path from typing import Iterable, TextIO -import sys -from contextlib import nullcontext import metomi.isodatetime.parsers import numpy @@ -345,14 +345,14 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: def odb2ascii(self, output_pattern: str, valid_times: Iterable[TimePoint]): """ Write all the observations to a MET ASCII file. - + If output_pattern contains a strftime-style pattern then the valid time will be used to replace the pattern. """ for t in valid_times: output = t.strftime(output_pattern) - if output == '-': + if output == "-": out_context = nullcontext(sys.stdout) else: out_context = open(output, "wt") diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py index 90e993814..477b855c8 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py @@ -55,7 +55,9 @@ def main(argv: list[str]): logging.basicConfig(level=logging.INFO) sys.tracebacklimit = 0 - PrepBomNci(args.system).odb2ascii(args.output, valid_times_iterator(args.valid_time)) + PrepBomNci(args.system).odb2ascii( + args.output, valid_times_iterator(args.valid_time) + ) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py index 97a5103bd..500ed02d7 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py @@ -15,7 +15,6 @@ import argparse import logging import sys -from contextlib import nullcontext from odb2.odb2 import PrepODB2Pattern, valid_times_iterator From b5ee78b12773a9de184196ef45ec6777d6256b62 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Thu, 18 Jun 2026 10:51:32 +1000 Subject: [PATCH 6/8] Add more comments to the opt file --- src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf | 11 +++++++++++ src/CSET/cset_workflow/site/nci-gadi.cylc | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf index 30495ae90..30461f232 100644 --- a/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf +++ b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf @@ -12,3 +12,14 @@ CSET_ENV_USE_MODULES = True CSET_ENV_SEPARATE_MET = False MODULES_PURGE = "" MODULES_LIST = "/g/data/access/ngm/modules/cset/26.4.0" + +# # Paths to local checkouts of repositories +# VERPY_DIR = "/path/to/verpy" +# CSET_DIR = "/path/to/cset" + +# # Metplus information +# # OBS_SYSTEM can be access_g{3,4}, access_c{3,4}_{ad,bn,dn,nq,ph,sy,vt} or custom +# METPLUS_OBS_SYSTEM = "access_g4" +# +# # If OBS_SYSTEM is custom, what path should be read +# CUSTOM_ODB_PATTERN = "/path/to/files/%Y%m%dT%H%MZ.odb2" diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index d2171247e..44f7904b6 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -30,7 +30,7 @@ platform = gadi_background [[[ environment ]]] PROJECT = {{ PROJECT }} - PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}{% endif %} + PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}/src{% endif %} {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %} [[METPLUS]] From 476d9fd740b54413969d4090452487a9975edc41 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Fri, 19 Jun 2026 11:27:18 +1000 Subject: [PATCH 7/8] Working test --- .../metplus_point_stat/file/nci/surface.conf | 4 +-- .../app/verpy_metloader/file/metloader.conf | 4 +-- .../app/verpy_metloader/rose-app.conf | 5 ++-- .../app/verpy_plot/bin/plot_stats.py | 7 +++-- .../app/verpy_plot/file/pointstat_cnt.rc | 22 +++++++++++++-- .../app/verpy_plot/rose-app.conf | 2 +- .../includes/metplus_point_stat.cylc | 27 ++++++++++--------- src/CSET/cset_workflow/site/nci-gadi.cylc | 9 +++++++ 8 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf index c518fca8d..be1634990 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf @@ -32,8 +32,8 @@ OBS_POINT_STAT_VAR2_NAME = rh2m OBS_POINT_STAT_VAR2_LEVELS = Z0 OBS_POINT_STAT_VAR2_THRESH = <60, >95 -OBS_POINT_STAT_WINDOW_BEGIN = -1800 -OBS_POINT_STAT_WINDOW_END = 1800 +OBS_POINT_STAT_WINDOW_BEGIN = -0 +OBS_POINT_STAT_WINDOW_END = 0 POINT_STAT_OUTPUT_FLAG_CNT = BOTH POINT_STAT_OUTPUT_FLAG_MPR = BOTH diff --git a/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf b/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf index 0eb8ed9a6..4a10f55b3 100644 --- a/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf +++ b/src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf @@ -17,9 +17,9 @@ max_fcrs = 240 # Elements used to pattern match files # MET output types as a comma separated list -stat_type = mpr +#stat_type = cnt # File prefix, often the name of the MET tool, defaults to "" -stream = point_stat +#stream = point_stat # Optional grid filter, defaults to "" #grid = G000 # Optional date filter, defaults to "" diff --git a/src/CSET/cset_workflow/app/verpy_metloader/rose-app.conf b/src/CSET/cset_workflow/app/verpy_metloader/rose-app.conf index c877bc443..6de439d92 100644 --- a/src/CSET/cset_workflow/app/verpy_metloader/rose-app.conf +++ b/src/CSET/cset_workflow/app/verpy_metloader/rose-app.conf @@ -3,8 +3,7 @@ default=${VERPY_DIR}/utils/bin/VerPyUtil_metloader.py ${METLOADER_CONF_FILE} [env] CONDA_VENV_LOCATION=${CONDA_METPLUS_VENV_LOCATION} -source=${CYLC_WORKFLOW_SHARE_DIR}/${MODEL_NAME}_${STREAM}_${VER_METHOD}_tar +source=${CYLC_WORKFLOW_SHARE_DIR}/${STREAM}_${VER_METHOD}_tar appdir=${CYLC_WORKFLOW_SHARE_DIR}/verpy_dbs output_table=${STREAM}_${VER_METHOD}_table -db_prefix=${VER_METHOD}_${STAT_TYPE} -# stream=${STREAM} +db_prefix=${VER_METHOD} diff --git a/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py b/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py index e5dd3e334..e425bd703 100755 --- a/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py +++ b/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py @@ -9,6 +9,7 @@ import argparse import json import os +import os.path import VerPy import VerPy.conf2opts @@ -41,9 +42,7 @@ def main(): options["start"] = args.start options["end"] = args.end options["expids"] = args.expids - options["source"] = ( - f"{os.getenv('TABLENAME')}@{os.getenv('DB_DIR')}/{os.getenv('DB_NAME')}" - ) + options["source"] = os.path.expandvars(options["source"]) print(f"Options Dictionary: {options}") VerPy.job.run(args.outdir, options) @@ -51,7 +50,7 @@ def main(): json_filename = f"{args.outdir}/{options['jobid']}/meta.json" print(f"writing metadata to json file: {json_filename}") metadata_dict = { - "title": "Metplus Point Stat plots", + "title": f"{options['system']} {options['type']} {options['output']}", "category": "Metplus plots", } with open(json_filename, "w") as jf: diff --git a/src/CSET/cset_workflow/app/verpy_plot/file/pointstat_cnt.rc b/src/CSET/cset_workflow/app/verpy_plot/file/pointstat_cnt.rc index 4a28e8b0f..68ae66419 100644 --- a/src/CSET/cset_workflow/app/verpy_plot/file/pointstat_cnt.rc +++ b/src/CSET/cset_workflow/app/verpy_plot/file/pointstat_cnt.rc @@ -2,11 +2,20 @@ comment: general stuff verbosity: 30 -[VerPy_sources] +[VerPy_source_cnt] comment: data sources type = cnt truth = Surface Obs system = MET +source = $TABLENAME@$DB_DIR/$DB_NAME + +[VerPy_source_mpr] +comment: data sources +type = mpr +truth = Surface Obs +system = MET +# Verpy doesn't like using the database for MET mpr analysis, read from the raw files +source = $CYLC_WORKFLOW_SHARE_DIR/point_stat_area_tar [VerPy_dates] comment: dates and times. Sourced via environment variables @@ -19,8 +28,17 @@ interp = NEAREST [VerPyPlot_plot1] comment = Lead Time plots -inherit = VerPy_settings, VerPy_sources, VerPy_dates, VerPy_plot_general +inherit = VerPy_settings, VerPy_source_cnt, VerPy_dates, VerPy_plot_general output = fcrseries jobid = cset_lead_times +metadata = cset_lead_times/index fcrs = [0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900, 2000, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900, 3000, 3100, 3200, 3300, 3400, 3500, 3600] times: [0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900, 2000, 2100, 2200, 2300] + +[VerPyPlot_errormap] +comment = Error Map Plots +inherit = VerPy_settings, VerPy_source_mpr, VerPy_dates, VerPy_plot_general +jobid = verpy_errormaps +metadata = verpy_errormaps/index +output = errormap +param = [(16, 1, -1.0)] diff --git a/src/CSET/cset_workflow/app/verpy_plot/rose-app.conf b/src/CSET/cset_workflow/app/verpy_plot/rose-app.conf index bdce12730..9813fa73b 100644 --- a/src/CSET/cset_workflow/app/verpy_plot/rose-app.conf +++ b/src/CSET/cset_workflow/app/verpy_plot/rose-app.conf @@ -11,4 +11,4 @@ EXPIDS=${MODEL_NAMES} DB_DIR=${CYLC_WORKFLOW_SHARE_DIR}/verpy_dbs TABLENAME=${STREAM}_${VER_METHOD}_table -DB_NAME=${VER_METHOD}_${STAT_TYPE}_${STAT_TYPE}.db +DB_NAME=${VER_METHOD}_${STAT_TYPE}.db diff --git a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc index e93499032..7d96eb65a 100644 --- a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc +++ b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc @@ -1,3 +1,5 @@ +{# METplus point_stat stat types to run #} +{% set POINT_STAT_TYPES = ["cnt", "mpr"] %} {% if RUN_METPLUS_POINT_STAT|default(False) %} [scheduling] @@ -25,9 +27,9 @@ """ {% endif %} R1/$ = """ - {% for model in models %} - cycle_complete => verpy_metloader_pointstat_m{{model["id"]}} - {% endfor %} + {% for STAT_TYPE in POINT_STAT_TYPES %} + cycle_complete => verpy_metloader_pointstat_{{STAT_TYPE}} + {% endfor %} VERPY_METLOADER:succeed-all => verpy_plot_pointstat => finish_website """ @@ -63,14 +65,14 @@ [[[environment]]] ROSE_TASK_APP = metplus_point_stat STAT = area - STAT_TYPE_LIST = cnt mpr + STAT_TYPE_LIST = {{ POINT_STAT_TYPES | join(' ') }} [[METPLUS_POINT_STAT_POSTPROC]] # Family grouping all metplus Post-processing inherit = METPLUS [[[environment]]] STAT = area - STAT_TYPE_LIST = cnt mpr + STAT_TYPE_LIST = {{ POINT_STAT_TYPES | join(' ') }} [[VERPY_METLOADER]] # Family grouping for all VerPy metloader tasks @@ -91,24 +93,25 @@ script = """ MODEL_NAME={{model["name"]}} POINT_STAT_DIR=${CYLC_TASK_SHARE_CYCLE_DIR}/Point_Stat_${MODEL_NAME} - mkdir -p ${CYLC_WORKFLOW_SHARE_DIR}/${MODEL_NAME}_point_stat_${STAT}_tar + mkdir -p ${CYLC_WORKFLOW_SHARE_DIR}/point_stat_${STAT}_tar for STAT_TYPE in ${STAT_TYPE_LIST}; do - gzip ${POINT_STAT_DIR}/*_${STAT_TYPE}.txt + gzip ${POINT_STAT_DIR}/*_${STAT_TYPE}.txt || true files_list=$(ls ${POINT_STAT_DIR}/*_${STAT_TYPE}.txt.gz | xargs -n 1 basename) - tar -rf ${CYLC_WORKFLOW_SHARE_DIR}/${MODEL_NAME}_point_stat_${STAT}_tar/point_stat_${CYLC_TASK_CYCLE_POINT}_${STAT_TYPE}.tar -C ${POINT_STAT_DIR} $files_list + tar -rf ${CYLC_WORKFLOW_SHARE_DIR}/point_stat_${STAT}_tar/${MODEL_NAME}_${CYLC_TASK_CYCLE_POINT}_${STAT_TYPE}.tar -C ${POINT_STAT_DIR} $files_list done """ +{% endfor %} - [[verpy_metloader_pointstat_m{{model["id"]}}]] +{% for STAT_TYPE in POINT_STAT_TYPES %} + [[verpy_metloader_pointstat_{{STAT_TYPE}}]] # Runs VerPy metloader utility to create the VerPy databases inherit = VERPY_METLOADER [[[environment]]] VERPY_DIR = {{VERPY_DIR}} + METLOADER_CONF_FILE = metloader.conf VER_METHOD = area - STAT_TYPE = cnt + STAT_TYPE = {{STAT_TYPE}} STREAM = point_stat - MODEL_NAME = {{model["name"]}} - METLOADER_CONF_FILE = metloader.conf {% endfor %} [[verpy_plot_pointstat]] diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index 44f7904b6..5d5f67308 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -46,6 +46,15 @@ [[[ environment ]]] ANALYSIS_LENGTH = {{ANALYSIS_LENGTH}} + [[METPLUS_POINT_STAT_POSTPROC]] + platform = gadi_background + + [[VERPY_METLOADER]] + platform = gadi_background + + [[verpy_plot_pointstat]] + platform = gadi_background + [[metplus_prep_obs]] [[[ environment ]]] OBS_TIMES = R{{(ANALYSIS_LENGTH | duration_as('h') | int) + 1}}/$CYLC_TASK_CYCLE_POINT/PT1H From 76d603b1771ec4ee933c973866a2ec71310c3740 Mon Sep 17 00:00:00 2001 From: Scott Wales Date: Fri, 19 Jun 2026 12:10:20 +1000 Subject: [PATCH 8/8] Teach verpy_plot about ODB2's rh2m name --- src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py b/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py index e425bd703..48b936293 100755 --- a/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py +++ b/src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py @@ -15,6 +15,13 @@ import VerPy.conf2opts +def AddOdb2Names(): + from VerPy.parameter import get_param_by_code + + param = get_param_by_code(88,1,-1) + param['short'].append('rh2m') + + def main(): """ Produce Verpy Plots. @@ -36,6 +43,8 @@ def main(): args = parser.parse_args() + AddOdb2Names() + opts_dicts, scard_dict = VerPy.conf2opts.conf2opts(args.conf) for options in opts_dicts: