diff --git a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf index f0f5dddf4..ee77d8c68 100644 --- a/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf +++ b/src/CSET/cset_workflow/app/metplus_ascii2nc/file/nci/ascii2nc.conf @@ -43,16 +43,8 @@ LEAD_SEQ = # https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info ### ASCII2NC_INPUT_DIR = {INPUT_BASE} -ASCII2NC_INPUT_TEMPLATE = *.ascii +ASCII2NC_INPUT_TEMPLATE = obs.{valid?fmt=%Y%m%dT%H%MZ}.ascii ASCII2NC_INPUT_FORMAT = met_point ASCII2NC_OUTPUT_DIR = {ENV[CYLC_WORKFLOW_SHARE_DIR]}/obs_nc ASCII2NC_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%dT%H}.nc - -### -# ASCII2NC Settings -# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#ascii2nc -### - -ASCII2NC_WINDOW_BEGIN = 0 -ASCII2NC_WINDOW_END = 0 diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf index 5d7b1880d..a3d6ed571 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/pointstat.conf @@ -36,10 +36,7 @@ INIT_TIME_FMT = %Y%m%dT%H INIT_BEG = {ENV[TASK_START_TIME]} INIT_END = {ENV[TASK_START_TIME]} INIT_INCREMENT = 1H -LEAD_SEQ = begin_end_incr(0,{ENV[FORECAST_LENGTH]},1) - -# Number of seconds to shift times in the fcst file (try half the time increment) -FCST_SHIFT = 1800 +LEAD_SEQ = begin_end_incr(4,{ENV[FORECAST_LENGTH]},1) ### # File I/O diff --git a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf index d61168298..e1c42c5b4 100644 --- a/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf +++ b/src/CSET/cset_workflow/app/metplus_point_stat/file/nci/surface.conf @@ -17,7 +17,7 @@ POINT_STAT_FCST_FILE_TYPE = NETCDF_NCCF POINT_STAT_MESSAGE_TYPE = "ADPSFC" FCST_POINT_STAT_VAR1_NAME = air_temperature -FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR1_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR1_THRESH = <=273, >273 OBS_POINT_STAT_VAR1_NAME = t2m @@ -25,7 +25,7 @@ OBS_POINT_STAT_VAR1_LEVELS = Z0 OBS_POINT_STAT_VAR1_THRESH = <=273, >273 FCST_POINT_STAT_VAR2_NAME = relative_humidity -FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S?shift={FCST_SHIFT}},*,*)" +FCST_POINT_STAT_VAR2_LEVELS = "({valid?fmt=%Y%m%d_%H%M%S},*,*)" FCST_POINT_STAT_VAR2_THRESH = <60, >95 OBS_POINT_STAT_VAR2_NAME = rh2m diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/bom.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/bom.py index c587c32fb..8993b17ec 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/bom.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/bom.py @@ -5,12 +5,13 @@ patterns. """ +from abc import abstractmethod from typing import Iterable from metomi.isodatetime.data import TimePoint from pandas import DataFrame -from .odb2 import PrepODB2, read_file, read_tarfile +from .odb2 import PrepODB2 # Valid bureau forecast systems BOM_SYSTEMS = ["access_g3", "access_g4"] @@ -32,20 +33,30 @@ def __init__(self, system: str): Args: system: BOM system to source observations from (e.g. 'access_c3_dn') """ + super().__init__() self.system = system + # Filter out observations rejected by OPS + self.odb_where += ( + " AND ops_report_flags@hdr = 0 AND ops_datum_flags.b0@body = 0" + ) + + @abstractmethod def read_c3_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: """Read C3 data pattern.""" raise NotImplementedError + @abstractmethod def read_c4_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: """Read C4 data pattern.""" raise NotImplementedError + @abstractmethod def read_g3_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: """Read G3 data pattern.""" raise NotImplementedError + @abstractmethod def read_g4_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: """Read G4 data pattern.""" raise NotImplementedError @@ -58,16 +69,16 @@ def read_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: type: observation type (ODB2 file name without the suffix) valid_time: time to load - Returns an iterable of ODB2 DataFrames + Returns an iterable of ODB2 DataFrames containing all the observations for the given type and time. """ if self.system.startswith("access_c3"): - yield from self.read_c3_type(type, valid_time) + return self.read_c3_type(type, valid_time) elif self.system.startswith("access_c4"): - yield from self.read_c4_type(type, valid_time) + return self.read_c4_type(type, valid_time) elif self.system.startswith("access_g3"): - yield from self.read_g3_type(type, valid_time) + return self.read_g3_type(type, valid_time) elif self.system.startswith("access_g4"): - yield from self.read_g4_type(type, valid_time) + return self.read_g4_type(type, valid_time) def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: """ @@ -76,7 +87,7 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: Args: valid_time: time to load - Returns an iterable of ODB2 DataFrames + Returns an iterable of ODB2 DataFrames containing all the observations for the given time. """ if self.system.startswith("access_c"): for type in access_c_types: @@ -107,25 +118,25 @@ def read_c3_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: domain = self.system.split("_")[-1] tarfile = f"/g/data/ig2/odb2/access_c3/%Y/%m/%Y%m%dT%H%MZ/%Y%m%dT%H%MZ_{domain}_odb2.tar.zst" obsfile = f"ukv_odb2/{type}.odb" - return read_tarfile(tarfile, obsfile, valid_time) + return self.read_tarfile(tarfile, obsfile, valid_time) def read_c4_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: """Read C4 data pattern.""" domain = self.system.split("_")[-1] tarfile = f"/g/data/ig2/odb2/access_c4/%Y/%m/%Y%m%dT%H%MZ/%Y%m%dT%H%MZ_{domain}_odb2.tgz" obsfile = f"ukv_odb2/{type}.odb" - return read_tarfile(tarfile, obsfile, valid_time) + return self.read_tarfile(tarfile, obsfile, valid_time) def read_g3_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: """Read G3 data pattern.""" pattern = ( "/g/data/ig2/odb2/access_g3/%Y/%m/%Y%m%dT%H%MZ/glm_odb2/{type}.odb.zst" ) - return read_file(pattern, valid_time) + return self.read_file(pattern, valid_time) def read_g4_type(self, type: str, valid_time: TimePoint) -> Iterable[DataFrame]: """Read G4 data pattern.""" pattern = ( "/g/data/ig2/odb2/access_g4/%Y/%m/%Y%m%dT%H%MZ/glm_odb2/{type}.odb.zst" ) - return read_file(pattern, valid_time) + return self.read_file(pattern, valid_time) diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py index ad7f45110..68c426907 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/odb2.py @@ -5,22 +5,30 @@ """ import functools +import io import json import logging +import shlex +import subprocess +import sys +import tempfile from abc import ABC, abstractmethod +from collections.abc import Iterable +from contextlib import nullcontext from glob import glob from pathlib import Path -from typing import Iterable, TextIO +from typing import BinaryIO, TextIO import metomi.isodatetime.parsers import numpy import pandas -import pyodc as odc from metomi.isodatetime.data import TimePoint from pandas import DataFrame try: - from compression import bz2, gzip, tarfile, zstd + import tarfile + + from compression import bz2, gzip, zstd except ImportError: import bz2 import gzip @@ -139,6 +147,7 @@ def get_level(obs: DataFrame) -> pandas.Series: plevel = obs["vertco_reference_1@body"].where( obs["vertco_type@body"].isin([1, 11, 15]), numpy.nan ) + plevel = plevel / 100 # Convert to hPa accumulated_fields = ["PRATE", "TSTM", "APCP", "NCPCP", "ACPCP"] if obs["name@varno"].isin(accumulated_fields).any(): @@ -200,17 +209,14 @@ def get_type(obs: DataFrame) -> pandas.Series: def odb2ascii_dataframe(obs: DataFrame) -> DataFrame: """ - Convert a DataFrame from pyodc to MET ASCII format. + Convert a DataFrame from ODB2 to MET ASCII format. args: - obs: ODB2 data loaded with pyodb + obs: ODB2 dataframe to convert Output format is described at https://metplus.readthedocs.io/projects/met/en/latest/Users_Guide/reformat_point.html#id9 """ - # QC filter for only active values - obs = obs.loc[(obs["report_status@hdr"] | obs["datum_status@body"]) == 1] - # Join in extra info obs = obs.join(varno_table(), on="varno@body") obs = obs.join(reporttype_table(), on="reportype@hdr") @@ -253,54 +259,35 @@ def write_ascii(dataframe: DataFrame, output: TextIO): ) -def read_tarfile(tarpath: str, path: str, valid_time: TimePoint) -> Iterable[DataFrame]: +def read_odb_sql( + stream: BinaryIO, columns: list[str], where: str | None = None +) -> DataFrame: """ - Read ODB2 data from a tarfile. - - Paths can use strftime templates which will be replaced by valid_time + Read ODB2 data using ODC CLI. Args: - tarpath: path of the tarfile - path: path inside the tarfile - valid_time: used for pattern replacement + stream: Binary stream containing the ODB2 data + columns: List of columns to select + where: Optional SQL-like filter to apply to the data """ - tarpath = valid_time.strftime(tarpath) - path = valid_time.strftime(path) - log.info("Reading %s:%s", tarpath, path) + sql = f"SELECT {', '.join(columns)}" + if where is not None: + sql += f" WHERE {where}" - with tarfile.open(tarpath, "r") as t: - f = t.extractfile(path) - yield from odc.read_odb(f) + with tempfile.NamedTemporaryFile(suffix=".odb2") as f: + f.write(stream.read()) + f.flush() + cmd = ["odc", "sql", "-i", f.name, sql] + log.info("Running command: %s", shlex.join(cmd)) + r = subprocess.run(cmd, stdout=subprocess.PIPE, text=True, check=True) -def read_file(pattern: str, valid_time: TimePoint) -> Iterable[DataFrame]: - """ - Read ODB2 data from a file, decompressing if required. + stream = io.StringIO(r.stdout) - Paths can use strftime templates which will be replaced by valid_time - Paths can use shell globs - Recognised extensions for decompression are .gz, .bz2, .zst + df = pandas.read_csv(stream, sep="\t", skipinitialspace=True) + df["statid@hdr"] = df["statid@hdr"].str.strip("'").str.strip() - Args: - path: path to open - valid_time: used for pattern replacement - """ - path = valid_time.strftime(pattern) - log.info("Reading %s", path) - - for p in glob(path): - if p.endswith(".gz"): - with gzip.open(p) as f: - yield from odc.read_odb(f) - elif p.endswith(".bz2"): - with bz2.open(p) as f: - yield from odc.read_odb(f) - elif p.endswith(".zst"): - with zstd.open(p) as f: - yield from odc.read_odb(f) - else: - with open(p, "rb") as f: - yield from odc.read_odb(f) + return df def valid_times_iterator( @@ -335,18 +322,105 @@ class PrepODB2(ABC): ... converter.odb2ascii(f, valid_times) """ + def __init__(self, odb_where: str | None = None): + """Construct the converter.""" + # ODB2 columns to read for each observation + self.odb_columns = [ + "vertco_type@body", + "vertco_reference_1@body", + "varno@body", + "statid@hdr", + "date@hdr", + "time@hdr", + "lat@hdr", + "lon@hdr", + "stalt@hdr", + "obsvalue@body", + "reportype@hdr", + ] + + # Optional SQL-like filter to apply to the data for QC filtering + self.odb_where = odb_where or "report_status@hdr = 1 AND datum_status@body = 1" + + def read_tarfile( + self, tarpath: str, path: str, valid_time: TimePoint + ) -> Iterable[DataFrame]: + """ + Read ODB2 data from a tarfile. + + Paths can use strftime templates which will be replaced by valid_time + + Args: + tarpath: path of the tarfile + path: path inside the tarfile + valid_time: used for pattern replacement + """ + tarpath = valid_time.strftime(tarpath) + path = valid_time.strftime(path) + log.info("Reading %s:%s", tarpath, path) + + with tarfile.open(tarpath, "r") as t: + f = t.extractfile(path) + yield read_odb_sql(f, self.odb_columns, self.odb_where) + + def read_file(self, pattern: str, valid_time: TimePoint) -> Iterable[DataFrame]: + """ + Read ODB2 data from a file, decompressing if required. + + Paths can use strftime templates which will be replaced by valid_time + Paths can use shell globs + Recognised extensions for decompression are .gz, .bz2, .zst + + Args: + pattern: path pattern to open + valid_time: used for pattern replacement + """ + path = valid_time.strftime(pattern) + log.info("Reading %s", path) + + for p in glob(path): + if p.endswith(".gz"): + with gzip.open(p) as f: + yield read_odb_sql(f, self.odb_columns, self.odb_where) + elif p.endswith(".bz2"): + with bz2.open(p) as f: + yield read_odb_sql(f, self.odb_columns, self.odb_where) + elif p.endswith(".zst"): + with zstd.open(p) as f: + yield read_odb_sql(f, self.odb_columns, self.odb_where) + else: + with open(p, "rb") as f: + yield read_odb_sql(f, self.odb_columns, self.odb_where) + @abstractmethod def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: - """Read ODB2 data.""" + """ + Read ODB2 data. + + Subclasses should implement this method to read ODB2 data for the given valid_time and return an iterable of DataFrames. + """ raise NotImplementedError - def odb2ascii(self, output: TextIO, valid_times: Iterable[TimePoint]): - """Write all the observations to a MET ASCII file.""" + def odb2ascii(self, output_pattern: str, valid_times: Iterable[TimePoint]): + """ + Write all the observations to a MET ASCII file. + + If output_pattern contains a strftime-style pattern then the valid time + will be used to replace the pattern. + """ for t in valid_times: + output = t.strftime(output_pattern) + + if output == "-": + out_context = nullcontext(sys.stdout) + else: + out_context = open(output, "wt") + log.info("Processing %s", t) - for obs in self.read_odb(t): - ascii = odb2ascii_dataframe(obs) - write_ascii(ascii, output) + with out_context as f: + for obs in self.read_odb(t): + ascii = odb2ascii_dataframe(obs) + write_ascii(ascii, f) class PrepODB2Pattern(PrepODB2): @@ -377,4 +451,4 @@ def read_odb(self, valid_time: TimePoint) -> Iterable[DataFrame]: Args: valid_time: Time to use in file patterns """ - return read_file(self.pattern, valid_time) + return self.read_file(self.pattern, valid_time) diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/test_odb2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/test_odb2.py index 1957ea60e..90e064e3b 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/test_odb2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/odb2/test_odb2.py @@ -1,16 +1,19 @@ """Tests for ODB2 to MET ASCII conversion.""" import io +from unittest.mock import patch import numpy import pandas from .odb2 import ( ASCII_COLUMNS, + PrepODB2Pattern, get_height, get_level, get_type, odb2ascii_dataframe, + read_odb_sql, write_ascii, ) @@ -157,3 +160,56 @@ def test_write_ascii(): write_ascii(ascii, output) expect = "ADPSFC\tDUMMY\t20010101_0100\t10\t20\t30\tt2m\tNA\t0\tNA\t40\n" assert output.getvalue() == expect + + +def test_read_odb(): + """Test read_odb.""" + with patch("subprocess.run") as mock_run: + mock_run.return_value.stdout = ( + "name@varno\tvertco_type@body\tvertco_reference_1@body\n" + ) + mock_run.return_value.returncode = 0 + df = read_odb_sql( + io.BytesIO(), ["name@varno", "vertco_type@body", "vertco_reference_1@body"] + ) + assert isinstance(df, pandas.DataFrame) + assert list(df.columns) == [ + "name@varno", + "vertco_type@body", + "vertco_reference_1@body", + ] + + +def test_PrepODB_read(tmp_path): + """Check the prepODB2 read_odb method returns a DataFrame with the expected columns.""" + obs = pandas.DataFrame( + [ + { + "reportype@hdr": 16001, + "report_status@hdr": 1, + "date@hdr": 20010101, + "time@hdr": 10000, + "datum_status@body": 1, + "varno@body": 39, + "statid@hdr": "DUMMY", + "lat@hdr": 10, + "lon@hdr": 20, + "stalt@hdr": 30, + "obsvalue@body": 40, + "vertco_type@body": 5, + "vertco_reference_1@body": 1, + }, + ] + ) + + (tmp_path / "test.odb2").write_text("") + pattern = str(tmp_path / "test.odb2") + + with patch("odb2.odb2.read_odb_sql") as mock_read_odb_sql: + mock_read_odb_sql.return_value = obs + + prep = PrepODB2Pattern(pattern) + df = next(prep.read_odb(pandas.Timestamp("20010101T0100Z"))) + assert isinstance(df, pandas.DataFrame) + + assert df["reportype@hdr"].iloc[0] == 16001 diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py index 2c3e1a191..ecba51bbc 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepBureauNCI.py @@ -6,17 +6,16 @@ Valid times can be either ISO timepoints or recurrences, and are used to replace any strftime patterns. Data is sourced from the mirror in the ig2 project, not all times are available. - ./prepODB2.py \ + ./prepBureauNCI.py \ --system access-c3-dn \ --valid-time 20010101T0000Z \ --valid-time R4/20010102T0000Z/PT6H \ - --output obs.ascii + --output obs.%Y%m%dT%H%MZ.ascii """ import argparse import logging import sys -from contextlib import nullcontext from odb2 import valid_times_iterator from odb2.bom import BOM_SYSTEMS, PrepBomNci @@ -54,15 +53,10 @@ def main(argv: list[str]): logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) - sys.tracebacklimit = 0 - if args.output == "-": - out_context = nullcontext(sys.stdout) - else: - out_context = open(args.output, "wt") - - with out_context as output: - PrepBomNci(args.system).odb2ascii(output, valid_times_iterator(args.valid_time)) + PrepBomNci(args.system).odb2ascii( + args.output, valid_times_iterator(args.valid_time) + ) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py index c021a261d..500ed02d7 100755 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/bin/prepODB2.py @@ -9,13 +9,12 @@ ./prepODB2.py /path/to/%Y/%m/%Y%m%dT%H%MZ/*.odb \ --valid-time 20010101T0000Z \ --valid-time R4/20010102T0000Z/PT6H \ - --output obs.ascii + --output obs.%Y%m%dT%H%MZ.ascii """ import argparse import logging import sys -from contextlib import nullcontext from odb2.odb2 import PrepODB2Pattern, valid_times_iterator @@ -71,16 +70,10 @@ def main(argv: list[str]): # Valid time unset, hopefully the pattern isn't using times args.valid_time = ["00010101T0000Z"] - if args.output == "-": - out_context = nullcontext(sys.stdout) - else: - out_context = open(args.output, "wt") - - with out_context as output: - for pattern in args.file: - PrepODB2Pattern(pattern).odb2ascii( - output, valid_times_iterator(args.valid_time) - ) + for pattern in args.file: + PrepODB2Pattern(pattern).odb2ascii( + args.output, valid_times_iterator(args.valid_time) + ) if __name__ == "__main__": diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf index 61375dd49..1fa93ef26 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-nci-gadi.conf @@ -1,4 +1,4 @@ [command] # Read in files from ig2 default = mkdir -p "$METPLUS_OBS_DIR" - = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" + = app_env_wrapper prepBureauNCI.py --system "$METPLUS_OBS_SYSTEM" --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" diff --git a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf index a47fea382..7ad4f9f67 100644 --- a/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf +++ b/src/CSET/cset_workflow/app/metplus_prep_obs/opt/rose-app-odb2.conf @@ -1,4 +1,4 @@ [command] # Read in files from ODB2 files by providing a strftime pattern default = mkdir -p "$METPLUS_OBS_DIR" - = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/${CYLC_TASK_CYCLE_POINT}.ascii" $CUSTOM_ODB2_PATTERN + = app_env_wrapper prepODB2.py --valid-time "$OBS_TIMES" --output "$METPLUS_OBS_DIR/obs.%Y%m%dT%H%MZ.ascii" $CUSTOM_ODB2_PATTERN diff --git a/src/CSET/cset_workflow/bin/app_env_wrapper b/src/CSET/cset_workflow/bin/app_env_wrapper index e67211101..a03dbaa26 100755 --- a/src/CSET/cset_workflow/bin/app_env_wrapper +++ b/src/CSET/cset_workflow/bin/app_env_wrapper @@ -11,7 +11,7 @@ set -euo pipefail if [[ $CSET_ENV_USE_MODULES == True ]]; then IFS_SAVE=$IFS IFS=' ' - if [[ "$MODULES_PURGE" ]]; then + if [[ "$MODULES_PURGE" == True ]]; then module purge fi for module in $MODULES_LIST; do diff --git a/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf new file mode 100644 index 000000000..dede8b080 --- /dev/null +++ b/src/CSET/cset_workflow/opt/rose-suite-nci-gadi.conf @@ -0,0 +1,25 @@ +[template variables] +SITE="nci-gadi" + +# Storage flags to add +NCI_STORAGE=["gdata/dp9"] + +# Where to store output website (view from ARE virtual desktop) +WEB_DIR="~/public_html/$CYLC_WORKFLOW_NAME" + +# Default module +CSET_ENV_USE_MODULES = True +CSET_ENV_SEPARATE_MET = False +MODULES_PURGE = False +MODULES_LIST = "/g/data/access/ngm/modules/cset/26.7.0" + +# # Paths to local checkouts of repositories +# VERPY_DIR = "/path/to/verpy" +# CSET_DIR = "/path/to/cset" + +# # Metplus information +# # OBS_SYSTEM can be access_g{3,4}, access_c{3,4}_{ad,bn,dn,nq,ph,sy,vt} or custom +# METPLUS_OBS_SYSTEM = "access_g4" +# +# # If OBS_SYSTEM is custom, what path should be read +# CUSTOM_ODB_PATTERN = "/path/to/files/%Y%m%dT%H%MZ.odb2" diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index 543ef4ea5..44f7904b6 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -2,10 +2,14 @@ # Site configuration for NCI Gadi # +# Using the opt file 'nci-gadi' is recommended to ensure the CSET module is +# loaded correctly +# # Important rose-suite.conf settings: # SITE = "nci-gadi" # Enable the site # PROJECT # NCI project code # VERPY_DIR # Path to Verpy source code +# NCI_STORAGE # List of NCI storage locations to use # METPLUS_OBS_SYSTEM # Where to source obs from # # Can be a Bureau forecast system e.g. # # 'access_c3_dn' or 'custom' @@ -14,16 +18,19 @@ # # Leave METPLUS_FCST_DIR and METPLUS_OBS_DIR unset +{% set STORAGE = [ + 'scratch/'~PROJECT, + 'gdata/'~PROJECT, + 'gdata/access', + 'gdata/hr22', +] + (NCI_STORAGE | default([])) %} + [runtime] [[root]] platform = gadi_background - init-script = """ - module use /g/data/access/ngm/modules - module load cset/26.4.0 - """ [[[ environment ]]] PROJECT = {{ PROJECT }} - PYTHONPATH = "{{VERPY_DIR}}" + PYTHONPATH = {{VERPY_DIR}}{% if CSET_DIR is defined %}:{{CSET_DIR}}/src{% endif %} {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %} [[METPLUS]] @@ -34,14 +41,14 @@ -q = normal -l ncpus = 1 -l mem = 4gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/ig2+scratch/dx2 + -l storage = {{ STORAGE | join('+') }}+gdata/ig2 -W umask = 0022 [[[ environment ]]] ANALYSIS_LENGTH = {{ANALYSIS_LENGTH}} [[metplus_prep_obs]] [[[ environment ]]] - OBS_TIMES = R{{ANALYSIS_LENGTH | duration_as('h') | int }}/$CYLC_TASK_CYCLE_POINT/PT1H + OBS_TIMES = R{{(ANALYSIS_LENGTH | duration_as('h') | int) + 1}}/$CYLC_TASK_CYCLE_POINT/PT1H {% if METPLUS_OBS_SYSTEM != "custom" %} # Forecast system to pull obs from (e.g. 'access_c3_dn') METPLUS_OBS_SYSTEM = {{METPLUS_OBS_SYSTEM}} @@ -60,7 +67,7 @@ -q = normal -l ncpus = 48 -l mem = 180gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2 + -l storage = {{ STORAGE | join('+') }} -W umask = 0022 [[[ environment ]]] BUNCH_POOL_SIZE = $PBS_NCPUS @@ -73,7 +80,7 @@ -q = normal -l ncpus = 48 -l mem = 180gb - -l storage = scratch/{{PROJECT}}+gdata/{{PROJECT}}+gdata/access+gdata/dp9+gdata/hr22+gdata/xp65+scratch/dx2 + -l storage = {{ STORAGE | join('+') }} -W umask = 0022 [[[ environment ]]] BUNCH_POOL_SIZE = $PBS_NCPUS