From 563501748041e42a0162524ef6aa73ab3574ad2d Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Wed, 24 Jun 2026 17:47:15 +0100 Subject: [PATCH 01/13] Add RMSE scores based vertical pressure profile recipe Include plotting of vertical profiles without sequence_coordinate for 1D cubes. --- .../includes/metplus_point_stat.cylc | 2 +- src/CSET/cset_workflow/site/nci-gadi.cylc | 2 +- src/CSET/operators/plot.py | 54 ++++++++++++------- .../generic_level_rmse_scores_profile.yaml | 36 +++++++++++++ 4 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml diff --git a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc index 00a01ab97..8880908e4 100644 --- a/src/CSET/cset_workflow/includes/metplus_point_stat.cylc +++ b/src/CSET/cset_workflow/includes/metplus_point_stat.cylc @@ -80,7 +80,7 @@ # Runs VerPy metloader utility to create the VerPy databases inherit = VERPY_METLOADER [[[environment]]] - VERPY_DIR = {{VERPY_DIR}} + VERPY_DIR = {{VERPY_DIR|default("")}} VER_METHOD = area STAT_TYPE = cnt STREAM = point_stat diff --git a/src/CSET/cset_workflow/site/nci-gadi.cylc b/src/CSET/cset_workflow/site/nci-gadi.cylc index 543ef4ea5..3e83a92f0 100644 --- a/src/CSET/cset_workflow/site/nci-gadi.cylc +++ b/src/CSET/cset_workflow/site/nci-gadi.cylc @@ -23,7 +23,7 @@ """ [[[ environment ]]] PROJECT = {{ PROJECT }} - PYTHONPATH = "{{VERPY_DIR}}" + PYTHONPATH = "{{VERPY_DIR|default("")}}" {% if RUN_METPLUS_GRID_STAT|default(False) or RUN_METPLUS_POINT_STAT|default(False) %} [[METPLUS]] diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index a4740a4c1..2f2a4eadf 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -2469,28 +2469,46 @@ def plot_vertical_line_series( vmin = min(x_levels) vmax = max(x_levels) - # Matching the slices (matching by seq coord point; it may happen that - # evaluated models do not cover the same seq coord range, hence matching - # necessary) - cube_iterables = _find_matched_slices(cubes, sequence_coordinate) + # Check if the cube has a sequence coordinate (e.g. time). If not, plot + # a single profile directly without iterating over a sequence. + has_sequence_coord = all(cube.coords(sequence_coordinate) for cube in cubes) - # Create a plot for each value of the sequence coordinate. - # Allowing for multiple cubes in a CubeList to be plotted in the same plot for - # similar sequence values. Passing a CubeList into the internal plotting function - # for similar values of the sequence coordinate. cube_slice can be an iris.cube.Cube - # or an iris.cube.CubeList. plot_index = [] - nplot = np.size(cubes[0].coord(sequence_coordinate).points) - for cubes_slice in cube_iterables: - # Format the coordinate value in a unit appropriate way. - seq_coord = cubes_slice[0].coord(sequence_coordinate) - plot_title, plot_filename = _set_title_and_filename( - seq_coord, nplot, recipe_title, filename - ) + if has_sequence_coord: + # Matching the slices (matching by seq coord point; it may happen that + # evaluated models do not cover the same seq coord range, hence matching + # necessary) + cube_iterables = _find_matched_slices(cubes, sequence_coordinate) + nplot = np.size(cubes[0].coord(sequence_coordinate).points) + for cubes_slice in cube_iterables: + # Format the coordinate value in a unit appropriate way. + seq_coord = cubes_slice[0].coord(sequence_coordinate) + plot_title, plot_filename = _set_title_and_filename( + seq_coord, nplot, recipe_title, filename + ) + + # Do the actual plotting. + _plot_and_save_vertical_line_series( + cubes_slice, + coords, + "realization", + plot_filename, + series_coordinate, + title=plot_title, + vmin=vmin, + vmax=vmax, + ) + plot_index.append(plot_filename) + else: + # 1D case: no sequence coordinate, plot a single profile. + plot_title = recipe_title + if filename: + plot_filename = filename + else: + plot_filename = f"{slugify(plot_title)}.png" - # Do the actual plotting. _plot_and_save_vertical_line_series( - cubes_slice, + cubes, coords, "realization", plot_filename, diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml new file mode 100644 index 000000000..f5b55db49 --- /dev/null +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -0,0 +1,36 @@ +category: Scores +title: "$VARNAME\nRMSE vertical profile between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME + as a vertical profile on pressure levels. The RMSE is calculated + pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point + and level, then aggregated over all time and horizontal grid points + to produce one RMSE value per pressure level. + + A larger RMSE implies a greater error than a smaller RMSE. An + RMSE of zero implies the two fields match exactly. + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + variable_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + level_constraint: + operator: constraints.generate_level_constraint + coordinate: pressure + levels: "*" + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_rmse + preserved_coordinates: ["pressure"] + + - operator: plot.plot_vertical_line_series + series_coordinate: pressure + + - operator: write.write_cube_to_nc + overwrite: True From 5fffbd4728d67e27c576d58789a629928c2fee01 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Wed, 24 Jun 2026 18:20:37 +0100 Subject: [PATCH 02/13] Calculate RMSE for each vertical level Preserve time as sequence coordinate to slide over. This required an addition to scoreswrappers.py that maps the xarray time dimensions with iris auxilliary time dimension requirements. The scoreswrapper was developed with help of Github copilot. --- src/CSET/operators/scoreswrappers.py | 2 +- .../verification/generic_level_rmse_scores_profile.yaml | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 977380250..bee585144 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -208,7 +208,7 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = A CubeList containing exactly two cubes: a base and an "other" model, this can be an analysis and the model. preserved_coordinates: list[str] | str | None, default is None. - The coordinates that you wish to preserve in the calculaiton of the + The coordinates (or xarray dimension names) that you wish to preserve in the calculaiton of the RMSE. For example if you want a map of each time you can preserve ["time","grid_latitude", "grid_longitude"] or if you want a time series you can preserve ["time"], if you want to collapse to a single value diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index f5b55db49..df745e6f9 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -4,8 +4,9 @@ description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point - and level, then aggregated over all time and horizontal grid points - to produce one RMSE value per pressure level. + over time on each vertical level, then aggregated over all time and horizontal grid points + to produce one RMSE value per pressure level. The RMSE is calculated in + a single operation for each level that includes the time dimension. A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero implies the two fields match exactly. @@ -27,10 +28,11 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["pressure"] + preserved_coordinates: ["time","pressure"] - operator: plot.plot_vertical_line_series series_coordinate: pressure + sequence_coordinate: time - operator: write.write_cube_to_nc overwrite: True From c56d2fa2bba5ac63e8f4040e97094fa804e8fa23 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 25 Jun 2026 12:21:03 +0100 Subject: [PATCH 03/13] Include aggregation period into title of vertical profile line plot For rmse_scores_profile when aggregating over an entire case study's time coordinate resulting in a single rmse profile. --- src/CSET/operators/plot.py | 30 ++++++++++++++++++- .../generic_level_rmse_scores_profile.yaml | 5 ++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/CSET/operators/plot.py b/src/CSET/operators/plot.py index 2f2a4eadf..d294f3ab5 100644 --- a/src/CSET/operators/plot.py +++ b/src/CSET/operators/plot.py @@ -2471,7 +2471,17 @@ def plot_vertical_line_series( # Check if the cube has a sequence coordinate (e.g. time). If not, plot # a single profile directly without iterating over a sequence. - has_sequence_coord = all(cube.coords(sequence_coordinate) for cube in cubes) + sequence_coords = [ + cube.coord(sequence_coordinate) + for cube in cubes + if cube.coords(sequence_coordinate) + ] + has_sequence_coord = len(sequence_coords) == len(cubes) and all( + np.size(coord.points) > 1 for coord in sequence_coords + ) + has_scalar_sequence_coord = len(sequence_coords) == len(cubes) and all( + np.size(coord.points) == 1 for coord in sequence_coords + ) plot_index = [] if has_sequence_coord: @@ -2499,6 +2509,24 @@ def plot_vertical_line_series( vmax=vmax, ) plot_index.append(plot_filename) + elif has_scalar_sequence_coord: + # Scalar sequence coordinate (typically aggregated time bounds): + # make one plot and include sequence period in title/filename. + plot_title, plot_filename = _set_title_and_filename( + sequence_coords[0], 1, recipe_title, filename + ) + + _plot_and_save_vertical_line_series( + cubes, + coords, + "realization", + plot_filename, + series_coordinate, + title=plot_title, + vmin=vmin, + vmax=vmax, + ) + plot_index.append(plot_filename) else: # 1D case: no sequence coordinate, plot a single profile. plot_title = recipe_title diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index df745e6f9..f027fa776 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -4,9 +4,8 @@ description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point - over time on each vertical level, then aggregated over all time and horizontal grid points - to produce one RMSE value per pressure level. The RMSE is calculated in - a single operation for each level that includes the time dimension. + and vertical level, then aggregated over all time and horizontal grid points + to produce one RMSE value per pressure level. A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero implies the two fields match exactly. From a0af0e3fd29097e3c3c18a3e8f357bcdeaae174f Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 25 Jun 2026 15:10:54 +0100 Subject: [PATCH 04/13] Wire the recipe into the cylc workflow --- .../meta/verification/rose-meta.conf | 8 ++++++++ src/CSET/cset_workflow/rose-suite.conf.example | 1 + src/CSET/loaders/verification.py | 18 ++++++++++++++++++ .../generic_level_rmse_scores_profile.yaml | 2 +- 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index ec57b3ad6..9d4a86b25 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -226,3 +226,11 @@ help=Usually this should be set to 0, but in some case it may be 1. type=integer compulsory=true sort-key=scores4b +sort-key=scores2 + +[template variables=SCORES_RMSE_VERTICAL_PROFILES] +ns=Verification/Scores +description=Create vertical profile RMSE plots for the specified level fields. +type=python_boolean +compulsory=true +sort-key=scores3 diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index a43f7a5f6..d17103f10 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -150,6 +150,7 @@ SCORES_ALL=False SCORES_CRPS_FOR_ENSEMBLE=False SCORES_RMSE_SPATIAL=False SCORES_RMSE_TIMESERIES=False +SCORES_RMSE_VERTICAL_PROFILES=True SCORES_SPATIAL_AB=False SCORES_SPATIAL_MAE=False SCORES_SPATIAL_RMSE=False diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 18623c243..1628df627 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -137,3 +137,21 @@ def load(conf: Config): model_ids=[model["id"]], aggregation=False, ) + + if conf.SCORES_RMSE_VERTICAL_PROFILES: + base_model = models[0] + for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): + yield RawRecipe( + recipe="generic_level_rmse_scores_profile.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index f027fa776..fcc7f67eb 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -27,7 +27,7 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["time","pressure"] + preserved_coordinates: ["pressure"] - operator: plot.plot_vertical_line_series series_coordinate: pressure From 09514c01980e2b6eafb5f0a8db3902f1a62277ff Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 25 Jun 2026 16:44:07 +0100 Subject: [PATCH 05/13] Select rmse profile for aggregated across all time steps in a case study In additional rmse profile for every single time step. --- .../meta/verification/rose-meta.conf | 15 +++++++++++++++ src/CSET/cset_workflow/rose-suite.conf.example | 1 + src/CSET/loaders/verification.py | 11 ++++++++++- .../generic_level_rmse_scores_profile.yaml | 11 +++++++---- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 9d4a86b25..ae7515b16 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -233,4 +233,19 @@ ns=Verification/Scores description=Create vertical profile RMSE plots for the specified level fields. type=python_boolean compulsory=true +trigger=template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: True; sort-key=scores3 + +[template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES] +ns=Verification/Scores +description=Produce one RMSE vertical profile per time step rather than a single + profile aggregated over the whole time coordinate. +help=When True, the RMSE is calculated at each grid point and level for every + time step, collapsing only the spatial (horizontal) dimensions. This produces + a sequence of vertical RMSE profile plots, one per time step. + + When False (default), the RMSE is collapsed over both horizontal space and + time, yielding a single vertical RMSE profile for the whole period. +type=python_boolean +compulsory=true +sort-key=scores4 diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index d17103f10..9d4932d5d 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -150,6 +150,7 @@ SCORES_ALL=False SCORES_CRPS_FOR_ENSEMBLE=False SCORES_RMSE_SPATIAL=False SCORES_RMSE_TIMESERIES=False +SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES=False SCORES_RMSE_VERTICAL_PROFILES=True SCORES_SPATIAL_AB=False SCORES_SPATIAL_MAE=False diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 1628df627..5d9f0f5bb 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -140,13 +140,22 @@ def load(conf: Config): if conf.SCORES_RMSE_VERTICAL_PROFILES: base_model = models[0] - for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): + # List of aggregation modes to generate recipes for + agg_modes = [("pressure", ["pressure"])] + if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: + agg_modes.append(("timeseries", ["time", "pressure"])) + + for (mode_name, preserved_coords), (model, field) in itertools.product( + agg_modes, itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS) + ): yield RawRecipe( recipe="generic_level_rmse_scores_profile.yaml", variables={ "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], + "PRESERVED_COORDS": preserved_coords, + "AGGREGATION_MODE": mode_name, "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index fcc7f67eb..6c3cf3fe4 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -1,11 +1,14 @@ category: Scores -title: "$VARNAME\nRMSE vertical profile between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nRMSE vertical profile ($AGGREGATION_MODE) between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point - and vertical level, then aggregated over all time and horizontal grid points - to produce one RMSE value per pressure level. + and vertical level, then aggregated over horizontal grid points. + + Aggregation mode: + - pressure: Aggregates over the entire time period, producing one RMSE profile + - timeseries: Preserves time, producing one RMSE profile per time step A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero implies the two fields match exactly. @@ -27,7 +30,7 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["pressure"] + preserved_coordinates: $PRESERVED_COORDS - operator: plot.plot_vertical_line_series series_coordinate: pressure From a65abfd9bd71b97a88cf28460a98f4487b90a8d7 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 29 Jun 2026 18:13:27 +0100 Subject: [PATCH 06/13] Add loader for vertical profile RMSE for every timestep In addition identifying base_model at start of loader. --- .../meta/verification/rose-meta.conf | 12 +++---- src/CSET/loaders/verification.py | 34 ++++++++++++++----- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index ae7515b16..8527f5ad7 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -231,21 +231,21 @@ sort-key=scores2 [template variables=SCORES_RMSE_VERTICAL_PROFILES] ns=Verification/Scores description=Create vertical profile RMSE plots for the specified level fields. + RMSE profiles are calculated at each grid point for all timesteps within a case study at each pressure level. type=python_boolean compulsory=true -trigger=template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: True; sort-key=scores3 [template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES] ns=Verification/Scores -description=Produce one RMSE vertical profile per time step rather than a single - profile aggregated over the whole time coordinate. +description=Also produce one RMSE vertical profile per time step. help=When True, the RMSE is calculated at each grid point and level for every time step, collapsing only the spatial (horizontal) dimensions. This produces - a sequence of vertical RMSE profile plots, one per time step. + a sequence of vertical RMSE profile plots, one per time step, in addition to + the profile that is aggregated across the whole case study. - When False (default), the RMSE is collapsed over both horizontal space and - time, yielding a single vertical RMSE profile for the whole period. + When False (default), only the aggregated vertical RMSE profile is produced. type=python_boolean compulsory=true +#trigger=template variables=SCORES_RMSE_VERTICAL_PROFILES: True; sort-key=scores4 diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 5d9f0f5bb..0c196d421 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -49,12 +49,14 @@ def load(conf: Config): """Yield recipes from the given workflow configuration.""" # Load a list of model detail dictionaries. models = get_models(conf.asdict()) + if not models: + return # Models are listed in order, so model 1 is the first element. + base_model = models[0] scores_spatial_methods = _get_scores_spatial_methods(conf) if scores_spatial_methods: # Produce 2D spatial plots of scores metrics. - base_model = models[0] for model, field, method, scores_method in itertools.product( models[1:], conf.SURFACE_FIELDS, @@ -101,7 +103,6 @@ def load(conf: Config): scores_timeseries_methods = _get_scores_timeseries_methods(conf) if scores_timeseries_methods: # Produce timeseries plots of scores metrics averaged over the domain for each case study. - base_model = models[0] for model, field, scores_method in itertools.product( models[1:], conf.SURFACE_FIELDS, scores_timeseries_methods ): @@ -139,23 +140,40 @@ def load(conf: Config): ) if conf.SCORES_RMSE_VERTICAL_PROFILES: - base_model = models[0] # List of aggregation modes to generate recipes for agg_modes = [("pressure", ["pressure"])] if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: agg_modes.append(("timeseries", ["time", "pressure"])) - for (mode_name, preserved_coords), (model, field) in itertools.product( - agg_modes, itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS) - ): + if conf.SCORES_RMSE_VERTICAL_PROFILES: + for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): yield RawRecipe( recipe="generic_level_rmse_scores_profile.yaml", variables={ "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], - "PRESERVED_COORDS": preserved_coords, - "AGGREGATION_MODE": mode_name, + "PRESERVED_COORDS": ["pressure"], + "AGGREGATION_MODE": "pressure", + "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + + if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: + for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): + yield RawRecipe( + recipe="generic_level_rmse_scores_profile.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "PRESERVED_COORDS": ["time", "pressure"], + "AGGREGATION_MODE": "timeseries", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA From 188d65b2191c3779ead60ab77432b92cc2968562 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 6 Jul 2026 17:25:34 +0100 Subject: [PATCH 07/13] Correct name for vertical profile at each timepoint to SEQUENCE --- src/CSET/loaders/verification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 0c196d421..233dad314 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -164,7 +164,7 @@ def load(conf: Config): aggregation=False, ) - if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: + if conf.SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE: for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): yield RawRecipe( recipe="generic_level_rmse_scores_profile.yaml", From 025275062f4ad31a8f18e0c58972a5d840e051bd Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 6 Jul 2026 17:42:58 +0100 Subject: [PATCH 08/13] Correctly name rmse profile to sequence for profiles at each timestep --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 8527f5ad7..69af224df 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -236,7 +236,7 @@ type=python_boolean compulsory=true sort-key=scores3 -[template variables=SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES] +[template variables=SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE] ns=Verification/Scores description=Also produce one RMSE vertical profile per time step. help=When True, the RMSE is calculated at each grid point and level for every From 23826c9169f43e8229261b89e79699eecda32197 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 16 Jul 2026 11:54:58 +0100 Subject: [PATCH 09/13] Add aggregation mode to recipe and loader to ensure clarity Ensures scientific clarity for basis of RMSE calculation for profiles. --- src/CSET/loaders/verification.py | 6 ++++-- .../verification/generic_level_rmse_scores_profile.yaml | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 233dad314..ffcac2112 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -145,6 +145,7 @@ def load(conf: Config): if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: agg_modes.append(("timeseries", ["time", "pressure"])) + # including AGGREGATION_MODE in the variables dictionary to allow for clearer labeling of plots. if conf.SCORES_RMSE_VERTICAL_PROFILES: for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): yield RawRecipe( @@ -154,7 +155,7 @@ def load(conf: Config): "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], "PRESERVED_COORDS": ["pressure"], - "AGGREGATION_MODE": "pressure", + "AGGREGATION_MODE": "Case-study RMSE", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA @@ -164,6 +165,7 @@ def load(conf: Config): aggregation=False, ) + # including AGGREGATION_MODE in the variables dictionary to allow for clearer labeling of plots. if conf.SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE: for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS): yield RawRecipe( @@ -173,7 +175,7 @@ def load(conf: Config): "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], "PRESERVED_COORDS": ["time", "pressure"], - "AGGREGATION_MODE": "timeseries", + "AGGREGATION_MODE": "Time-step RMSE", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index 6c3cf3fe4..0c4a75517 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nRMSE vertical profile ($AGGREGATION_MODE) between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nRMSE vertical profile between $OTHER_MODEL and $BASE_MODEL ($AGGREGATION_MODE)" description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated @@ -7,8 +7,8 @@ description: | and vertical level, then aggregated over horizontal grid points. Aggregation mode: - - pressure: Aggregates over the entire time period, producing one RMSE profile - - timeseries: Preserves time, producing one RMSE profile per time step + - Case-study RMSE: Single RMSE profile calculated using all time points in the case study. + - Time-step RMSE: RMSE profiles calculated at each time point in a case study. A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero implies the two fields match exactly. From 4bc6ad0a6e8a697a5d2aacc0b2f9d28c30c23a26 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 16 Jul 2026 12:13:14 +0100 Subject: [PATCH 10/13] Set scores to False in rose-suite.conf_example --- src/CSET/cset_workflow/rose-suite.conf.example | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 9d4932d5d..32c011bad 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -150,7 +150,9 @@ SCORES_ALL=False SCORES_CRPS_FOR_ENSEMBLE=False SCORES_RMSE_SPATIAL=False SCORES_RMSE_TIMESERIES=False -SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES=False +SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE=False +SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE=False +SCORES_RMSE_VERTICAL_PROFILES=False SCORES_RMSE_VERTICAL_PROFILES=True SCORES_SPATIAL_AB=False SCORES_SPATIAL_MAE=False From 4f7a354e3c9d33e455c3b87374e34758c86ff544 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Thu, 16 Jul 2026 12:30:39 +0100 Subject: [PATCH 11/13] split scores sections into spatial, timeseries and profile. --- .../meta/verification/rose-meta.conf | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 69af224df..c79183261 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -77,7 +77,7 @@ trigger=template variables=SCORES_SPATIAL_RMSE: False; template variables=SCORES_TIMESERIES_AB: False; template variables=SCORES_TIMESERIES_PC: False; compulsory=true -sort-key=scores1 +sort-key=scoresspatial1 # Scores spatial plots @@ -228,16 +228,22 @@ compulsory=true sort-key=scores4b sort-key=scores2 +[Verification/Scores vertical profiles] +ns=Verification/Scores vertical profiles +sort-key=scoresvertical +title=Scores vertical profiles + +[Verification/Scores vertical profiles] [template variables=SCORES_RMSE_VERTICAL_PROFILES] -ns=Verification/Scores +ns=Verification/Scores vertical profiles description=Create vertical profile RMSE plots for the specified level fields. RMSE profiles are calculated at each grid point for all timesteps within a case study at each pressure level. type=python_boolean compulsory=true -sort-key=scores3 +sort-key=scoresprofile1 [template variables=SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE] -ns=Verification/Scores +ns=Verification/Scores vertical profiles description=Also produce one RMSE vertical profile per time step. help=When True, the RMSE is calculated at each grid point and level for every time step, collapsing only the spatial (horizontal) dimensions. This produces @@ -248,4 +254,4 @@ help=When True, the RMSE is calculated at each grid point and level for every type=python_boolean compulsory=true #trigger=template variables=SCORES_RMSE_VERTICAL_PROFILES: True; -sort-key=scores4 +sort-key=scoresprofile2 From 18e3062f73856b1eb148183cf67583895bd88ecb Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Mon, 20 Jul 2026 11:25:10 +0100 Subject: [PATCH 12/13] update science background for rmse vertical profile recipe. --- .../generic_level_rmse_scores_profile.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml index 0c4a75517..d4b2788d7 100644 --- a/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml +++ b/src/CSET/recipes/verification/generic_level_rmse_scores_profile.yaml @@ -3,8 +3,16 @@ title: "$VARNAME\nRMSE vertical profile between $OTHER_MODEL and $BASE_MODEL ($A description: | Extracts and plots the Root Mean Square Error (RMSE) of $VARNAME as a vertical profile on pressure levels. The RMSE is calculated - pairwise between $BASE_MODEL and $OTHER_MODEL at each grid point - and vertical level, then aggregated over horizontal grid points. + pairwise between $BASE_MODEL and $OTHER_MODEL for each pressure + level field utilising the preserved coordinates in the scores package. + + By adding a preserved coordinate to the recipe, the RMSE can be calculated for each + time step in a case study or for all time steps combined. + + References: + https://scores.readthedocs.io/en/stable/ + https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse + Aggregation mode: - Case-study RMSE: Single RMSE profile calculated using all time points in the case study. From 6e01005289ab2ad8769da9523138e6ef680a5b97 Mon Sep 17 00:00:00 2001 From: James Frost Date: Tue, 4 Aug 2026 09:30:09 +0100 Subject: [PATCH 13/13] Delete code resurrected by rebase Co-authored-by: James Frost --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 1 - src/CSET/cset_workflow/rose-suite.conf.example | 2 -- src/CSET/loaders/verification.py | 6 ------ 3 files changed, 9 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index c79183261..0b5b037d3 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -226,7 +226,6 @@ help=Usually this should be set to 0, but in some case it may be 1. type=integer compulsory=true sort-key=scores4b -sort-key=scores2 [Verification/Scores vertical profiles] ns=Verification/Scores vertical profiles diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 32c011bad..636747ce3 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -151,9 +151,7 @@ SCORES_CRPS_FOR_ENSEMBLE=False SCORES_RMSE_SPATIAL=False SCORES_RMSE_TIMESERIES=False SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE=False -SCORES_RMSE_VERTICAL_PROFILES_SEQUENCE=False SCORES_RMSE_VERTICAL_PROFILES=False -SCORES_RMSE_VERTICAL_PROFILES=True SCORES_SPATIAL_AB=False SCORES_SPATIAL_MAE=False SCORES_SPATIAL_RMSE=False diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index ffcac2112..7b99e75ec 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -139,12 +139,6 @@ def load(conf: Config): aggregation=False, ) - if conf.SCORES_RMSE_VERTICAL_PROFILES: - # List of aggregation modes to generate recipes for - agg_modes = [("pressure", ["pressure"])] - if conf.SCORES_RMSE_VERTICAL_PROFILES_TIMESERIES: - agg_modes.append(("timeseries", ["time", "pressure"])) - # including AGGREGATION_MODE in the variables dictionary to allow for clearer labeling of plots. if conf.SCORES_RMSE_VERTICAL_PROFILES: for model, field in itertools.product(models[1:], conf.PRESSURE_LEVEL_FIELDS):