Add verpy errormaps

src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf

@@ -43,16 +43,8 @@ LEAD_SEQ =
 # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info
 ###
 ASCII2NC_INPUT_DIR = {INPUT_BASE}
-ASCII2NC_INPUT_TEMPLATE = *.ascii
+ASCII2NC_INPUT_TEMPLATE = obs.{valid?fmt=%Y%m%dT%H%MZ}.ascii
 ASCII2NC_INPUT_FORMAT = met_point
 
 ASCII2NC_OUTPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/obs_nc
 ASCII2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%dT%H}.nc
-
-###
-# ASCII2NC Settings
-# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc
-###
-
-ASCII2NC_WINDOW_BEGIN = 0
-ASCII2NC_WINDOW_END = 0

src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf

@@ -36,10 +36,7 @@ INIT_TIME_FMT = %Y%m%dT%H
 INIT_BEG = {ENV[TASK_START_TIME]}
 INIT_END = {ENV[TASK_START_TIME]}
 INIT_INCREMENT = 1H
-LEAD_SEQ = begin_end_incr(0,{ENV[FORECAST_LENGTH]},1)
-
-# Number of seconds to shift times in the fcst file (try half the time increment)
-FCST_SHIFT = 1800
+LEAD_SEQ = begin_end_incr(4,{ENV[FORECAST_LENGTH]},1)
 
 ###
 # File I/O

src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf

@@ -17,22 +17,23 @@ POINT_STAT_FCST_FILE_TYPE = NETCDF_NCCF
 POINT_STAT_MESSAGE_TYPE = "ADPSFC"
 
 FCST_POINT_STAT_VAR1_NAME = air_temperature
-FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)"
+FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)"
 FCST_POINT_STAT_VAR1_THRESH = <=273, >273
 
 OBS_POINT_STAT_VAR1_NAME = t2m
 OBS_POINT_STAT_VAR1_LEVELS = Z0
 OBS_POINT_STAT_VAR1_THRESH = <=273, >273
 
 FCST_POINT_STAT_VAR2_NAME = relative_humidity
-FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)"
+FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)"
 FCST_POINT_STAT_VAR2_THRESH =  <60, >95
 
 OBS_POINT_STAT_VAR2_NAME = rh2m
 OBS_POINT_STAT_VAR2_LEVELS = Z0
 OBS_POINT_STAT_VAR2_THRESH = <60, >95
 
-OBS_POINT_STAT_WINDOW_BEGIN = -1800
-OBS_POINT_STAT_WINDOW_END = 1800
+OBS_POINT_STAT_WINDOW_BEGIN = -0
+OBS_POINT_STAT_WINDOW_END = 0
 
 POINT_STAT_OUTPUT_FLAG_CNT = BOTH
+POINT_STAT_OUTPUT_FLAG_MPR = BOTH

src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py

@@ -7,7 +7,9 @@
 import functools
 import json
 import logging
+import sys
 from abc import ABC, abstractmethod
+from contextlib import nullcontext
 from glob import glob
 from pathlib import Path
 from typing import Iterable, TextIO
@@ -340,13 +342,26 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]:
         """Read ODB2 data."""
         raise NotImplementedError
 
-    def odb2ascii(self, output: TextIO, valid_times: Iterable[TimePoint]):
-        """Write all the observations to a MET ASCII file."""
+    def odb2ascii(self, output_pattern: str, valid_times: Iterable[TimePoint]):
+        """
+        Write all the observations to a MET ASCII file.
+
+        If output_pattern contains a strftime-style pattern then the valid time
+        will be used to replace the pattern.
+        """
         for t in valid_times:
+            output = t.strftime(output_pattern)
+
+            if output == "-":
+                out_context = nullcontext(sys.stdout)
+            else:
+                out_context = open(output, "wt")
+
             log.info("Processing %s", t)
-            for obs in self.read_odb(t):
-                ascii = odb2ascii_dataframe(obs)
-                write_ascii(ascii, output)
+            with out_context as f:
+                for obs in self.read_odb(t):
+                    ascii = odb2ascii_dataframe(obs)
+                    write_ascii(ascii, f)
 
 
 class PrepODB2Pattern(PrepODB2):

src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py

@@ -6,17 +6,16 @@
 Valid times can be either ISO timepoints or recurrences, and are used to replace any strftime patterns.
 Data is sourced from the mirror in the ig2 project, not all times are available.
 
-    ./prepODB2.py \
+    ./prepBureauNCI.py \
             --system access-c3-dn \
             --valid-time 20010101T0000Z \
             --valid-time R4/20010102T0000Z/PT6H \
-            --output obs.ascii
+            --output obs.%Y%m%dT%H%MZ.ascii
 """
 
 import argparse
 import logging
 import sys
-from contextlib import nullcontext
 
 from odb2 import valid_times_iterator
 from odb2.bom import BOM_SYSTEMS, PrepBomNci
@@ -56,13 +55,9 @@ def main(argv: list[str]):
         logging.basicConfig(level=logging.INFO)
         sys.tracebacklimit = 0
 
-    if args.output == "-":
-        out_context = nullcontext(sys.stdout)
-    else:
-        out_context = open(args.output, "wt")
-
-    with out_context as output:
-        PrepBomNci(args.system).odb2ascii(output, valid_times_iterator(args.valid_time))
+    PrepBomNci(args.system).odb2ascii(
+        args.output, valid_times_iterator(args.valid_time)
+    )
 
 
 if __name__ == "__main__":

src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py

@@ -9,13 +9,12 @@
     ./prepODB2.py /path/to/%Y/%m/%Y%m%dT%H%MZ/*.odb \
             --valid-time 20010101T0000Z \
             --valid-time R4/20010102T0000Z/PT6H \
-            --output obs.ascii
+            --output obs.%Y%m%dT%H%MZ.ascii
 """
 
 import argparse
 import logging
 import sys
-from contextlib import nullcontext
 
 from odb2.odb2 import PrepODB2Pattern, valid_times_iterator
 
@@ -71,16 +70,10 @@ def main(argv: list[str]):
         # Valid time unset, hopefully the pattern isn't using times
         args.valid_time = ["00010101T0000Z"]
 
-    if args.output == "-":
-        out_context = nullcontext(sys.stdout)
-    else:
-        out_context = open(args.output, "wt")
-
-    with out_context as output:
-        for pattern in args.file:
-            PrepODB2Pattern(pattern).odb2ascii(
-                output, valid_times_iterator(args.valid_time)
-            )
+    for pattern in args.file:
+        PrepODB2Pattern(pattern).odb2ascii(
+            args.output, valid_times_iterator(args.valid_time)
+        )
 
 
 if __name__ == "__main__":

src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf

@@ -1,4 +1,4 @@
 [command]
 # Read in files from ig2
 default = mkdir -p "$METPLUS_OBS_DIR"
-        = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii"
+        = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii"

src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf

@@ -1,4 +1,4 @@
 [command]
 # Read in files from ODB2 files by providing a strftime pattern
 default = mkdir -p "$METPLUS_OBS_DIR"
-        = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" $CUSTOM_ODB2_PATTERN
+        = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" $CUSTOM_ODB2_PATTERN

src/CSET/cset_workflow/app/verpy_metloader/file/metloader.conf

@@ -17,9 +17,9 @@ max_fcrs = 240
 
 # Elements used to pattern match files
 # MET output types as a comma separated list
-stat_type = cnt
+#stat_type = cnt
 # File prefix, often the name of the MET tool, defaults to ""
-stream = point_stat
+#stream = point_stat
 # Optional grid filter, defaults to ""
 #grid = G000
 # Optional date filter, defaults to ""

src/CSET/cset_workflow/app/verpy_metloader/rose-app.conf

@@ -3,8 +3,7 @@ default=${VERPY_DIR}/utils/bin/VerPyUtil_metloader.py ${METLOADER_CONF_FILE}
 
 [env]
 CONDA_VENV_LOCATION=${CONDA_METPLUS_VENV_LOCATION}
-source=${CYLC_WORKFLOW_SHARE_DIR}/${MODEL_NAME}_${STREAM}_${VER_METHOD}_tar
+source=${CYLC_WORKFLOW_SHARE_DIR}/${STREAM}_${VER_METHOD}_tar
 appdir=${CYLC_WORKFLOW_SHARE_DIR}/verpy_dbs
 output_table=${STREAM}_${VER_METHOD}_table
-db_prefix=${VER_METHOD}_${STAT_TYPE}
-# stream=${STREAM}
+db_prefix=${VER_METHOD}

src/CSET/cset_workflow/app/verpy_plot/bin/plot_stats.py

@@ -9,11 +9,19 @@
 import argparse
 import json
 import os
+import os.path
 
 import VerPy
 import VerPy.conf2opts
 
 
+def AddOdb2Names():
+    from VerPy.parameter import get_param_by_code
+
+    param = get_param_by_code(88,1,-1)
+    param['short'].append('rh2m')
+
+
 def main():
     """
     Produce Verpy Plots.
@@ -35,23 +43,23 @@
 
     args = parser.parse_args()
 
+    AddOdb2Names()
+
     opts_dicts, scard_dict = VerPy.conf2opts.conf2opts(args.conf)
 
     for options in opts_dicts:
         options["start"] = args.start
         options["end"] = args.end
         options["expids"] = args.expids
-        options["source"] = (
-            f"{os.getenv('TABLENAME')}@{os.getenv('DB_DIR')}/{os.getenv('DB_NAME')}"
-        )
+        options["source"] = os.path.expandvars(options["source"])
         print(f"Options Dictionary: {options}")
         VerPy.job.run(args.outdir, options)
 
         # Create a json metadata file in outdir
         json_filename = f"{args.outdir}/{options['jobid']}/meta.json"
         print(f"writing metadata to json file: {json_filename}")
         metadata_dict = {
-            "title": "Metplus Point Stat plots",
+            "title": f"{options['system']} {options['type']} {options['output']}",
             "category": "Metplus plots",
         }
         with open(json_filename, "w") as jf:

src/CSET/cset_workflow/app/verpy_plot/file/pointstat_cnt.rc

@@ -2,11 +2,20 @@
 comment: general stuff
 verbosity: 30
 
-[VerPy_sources]
+[VerPy_source_cnt]
 comment: data sources
 type = cnt
 truth = Surface Obs
 system = MET
+source = $TABLENAME@$DB_DIR/$DB_NAME
+
+[VerPy_source_mpr]
+comment: data sources
+type = mpr
+truth = Surface Obs
+system = MET
+# Verpy doesn't like using the database for MET mpr analysis, read from the raw files
+source = $CYLC_WORKFLOW_SHARE_DIR/point_stat_area_tar
 
 [VerPy_dates]
 comment: dates and times. Sourced via environment variables
@@ -19,8 +28,17 @@ interp = NEAREST
 
 [VerPyPlot_plot1]
 comment = Lead Time plots
-inherit = VerPy_settings, VerPy_sources, VerPy_dates, VerPy_plot_general
+inherit = VerPy_settings, VerPy_source_cnt, VerPy_dates, VerPy_plot_general
 output = fcrseries
 jobid = cset_lead_times
+metadata = cset_lead_times/index
 fcrs = [0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900, 2000, 2100, 2200, 2300, 2400, 2500, 2600, 2700, 2800, 2900, 3000, 3100, 3200, 3300, 3400, 3500, 3600]
 times: [0, 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900, 2000, 2100, 2200, 2300]
+
+[VerPyPlot_errormap]
+comment = Error Map Plots
+inherit = VerPy_settings, VerPy_source_mpr, VerPy_dates, VerPy_plot_general
+jobid = verpy_errormaps
+metadata = verpy_errormaps/index
+output = errormap
+param =  [(16, 1, -1.0)]

src/CSET/cset_workflow/app/verpy_plot/rose-app.conf

@@ -11,4 +11,4 @@ EXPIDS=${MODEL_NAMES}
 
 DB_DIR=${CYLC_WORKFLOW_SHARE_DIR}/verpy_dbs
 TABLENAME=${STREAM}_${VER_METHOD}_table
-DB_NAME=${VER_METHOD}_${STAT_TYPE}_${STAT_TYPE}.db
+DB_NAME=${VER_METHOD}_${STAT_TYPE}.db

src/CSET/cset_workflow/includes/metplus_point_stat.cylc

@@ -1,3 +1,5 @@
+{# METplus point_stat stat types to run #}
+{% set POINT_STAT_TYPES = ["cnt", "mpr"] %}
 
 {% if RUN_METPLUS_POINT_STAT|default(False) %}
 [scheduling]
@@ -25,9 +27,9 @@
                 """
     {% endif %}
     R1/$ = """
-        {% for model in models %}
-            cycle_complete => verpy_metloader_pointstat_m{{model["id"]}}
-        {% endfor %}
+    {% for STAT_TYPE in POINT_STAT_TYPES %}
+        cycle_complete => verpy_metloader_pointstat_{{STAT_TYPE}}
+    {% endfor %}
         VERPY_METLOADER:succeed-all => verpy_plot_pointstat => finish_website
         """
 
@@ -63,14 +65,14 @@
         [[[environment]]]
             ROSE_TASK_APP = metplus_point_stat
             STAT = area
-            STAT_TYPE_LIST = cnt
+            STAT_TYPE_LIST = {{ POINT_STAT_TYPES | join(' ') }}
 
     [[METPLUS_POINT_STAT_POSTPROC]]
     # Family grouping all metplus Post-processing
     inherit = METPLUS
         [[[environment]]]
             STAT = area
-            STAT_TYPE_LIST = cnt
+            STAT_TYPE_LIST = {{ POINT_STAT_TYPES | join(' ') }}
 
     [[VERPY_METLOADER]]
     # Family grouping for all VerPy metloader tasks
@@ -91,24 +93,25 @@
         script = """
             MODEL_NAME={{model["name"]}}
             POINT_STAT_DIR=${CYLC_TASK_SHARE_CYCLE_DIR}/Point_Stat_${MODEL_NAME}
-            mkdir -p ${CYLC_WORKFLOW_SHARE_DIR}/${MODEL_NAME}_point_stat_${STAT}_tar
+            mkdir -p ${CYLC_WORKFLOW_SHARE_DIR}/point_stat_${STAT}_tar
             for STAT_TYPE in ${STAT_TYPE_LIST}; do
-                gzip ${POINT_STAT_DIR}/*_${STAT_TYPE}.txt
+                gzip ${POINT_STAT_DIR}/*_${STAT_TYPE}.txt || true
                 files_list=$(ls ${POINT_STAT_DIR}/*_${STAT_TYPE}.txt.gz | xargs -n 1 basename)
-                tar -rf ${CYLC_WORKFLOW_SHARE_DIR}/${MODEL_NAME}_point_stat_${STAT}_tar/point_stat_${CYLC_TASK_CYCLE_POINT}_${STAT_TYPE}.tar -C ${POINT_STAT_DIR} $files_list
+                tar -rf ${CYLC_WORKFLOW_SHARE_DIR}/point_stat_${STAT}_tar/${MODEL_NAME}_${CYLC_TASK_CYCLE_POINT}_${STAT_TYPE}.tar -C ${POINT_STAT_DIR} $files_list
             done
         """
+{% endfor %}
 
-    [[verpy_metloader_pointstat_m{{model["id"]}}]]
+{% for STAT_TYPE in POINT_STAT_TYPES %}
+    [[verpy_metloader_pointstat_{{STAT_TYPE}}]]
     # Runs VerPy metloader utility to create the VerPy databases
         inherit = VERPY_METLOADER
         [[[environment]]]
             VERPY_DIR = {{VERPY_DIR}}
+            METLOADER_CONF_FILE = metloader.conf
             VER_METHOD = area
-            STAT_TYPE = cnt
+            STAT_TYPE = {{STAT_TYPE}}
             STREAM = point_stat
-            MODEL_NAME = {{model["name"]}}
-            METLOADER_CONF_FILE = metloader.conf
 {% endfor %}
 
     [[verpy_plot_pointstat]]

src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf

@@ -0,0 +1,25 @@
+[template variables]
+SITE="nci-gadi"
+
+# Storage flags to add
+NCI_STORAGE=["gdata/dp9"]
+
+# Where to store output website (view from ARE virtual desktop)
+WEB_DIR="~/public_html/$CYLC_WORKFLOW_NAME"
+
+# Default module
+CSET_ENV_USE_MODULES = True
+CSET_ENV_SEPARATE_MET = False
+MODULES_PURGE = ""
+MODULES_LIST = "/g/data/access/ngm/modules/cset/26.4.0"
+
+# # Paths to local checkouts of repositories
+# VERPY_DIR = "/path/to/verpy"
+# CSET_DIR = "/path/to/cset"
+
+# # Metplus information
+# # OBS_SYSTEM can be access_g{3,4}, access_c{3,4}_{ad,bn,dn,nq,ph,sy,vt} or custom
+# METPLUS_OBS_SYSTEM = "access_g4"
+#
+# # If OBS_SYSTEM is custom, what path should be read
+# CUSTOM_ODB_PATTERN = "/path/to/files/%Y%m%dT%H%MZ.odb2"