From 09490d1f40a714967f9ace5ac8f3f65cdc6c88f3 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 24 Jun 2026 15:53:48 +0100 Subject: [PATCH 001/117] Loaders for new Scores metrics #2214 --- src/CSET/loaders/verification.py | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 88d4cc3f4..46b2c04bb 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -25,6 +25,70 @@ def load(conf: Config): models = get_models(conf.asdict()) # Models are listed in order, so model 1 is the first element. + if any(conf.SCORES_SPATIAL_DIFFERENCE): + base_model = models[0] + difference_methods = [] + if conf.SCORES_SPATIAL_DIFFERENCE[0]: + difference_methods.append("RMSE") + if conf.SCORES_SPATIAL_DIFFERENCE[1]: + difference_methods.append("additive_bias") + if conf.SCORES_SPATIAL_DIFFERENCE[2]: + difference_methods.append("MAE") + # if conf.SCORES_SPATIAL_DIFFERENCE[3]: + # difference_methods.append("correlation_pearsonr") + for model, field, method, difference_method in itertools.product( + models[1:], + conf.SURFACE_FIELDS, + conf.SPATIAL_SURFACE_FIELD_METHOD, + difference_methods, + ): + yield RawRecipe( + recipe="surface_difference_scores.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "METHOD": method, + "DIFFERENCE_METHOD": difference_method, + "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + + if any(conf.SCORES_DIFFERENCE_TIMESERIES): + base_model = models[0] + difference_methods = [] + if conf.SCORES_DIFFERENCE_TIMESERIES[0]: + difference_methods.append("RMSE") + if conf.SCORES_DIFFERENCE_TIMESERIES[1]: + difference_methods.append("additive_bias") + if conf.SCORES_DIFFERENCE_TIMESERIES[2]: + difference_methods.append("MAE") + if conf.SCORES_DIFFERENCE_TIMESERIES[3]: + difference_methods.append("correlation_pearsonr") + for model, field, difference_method in itertools.product( + models[1:], conf.SURFACE_FIELDS, difference_methods + ): + yield RawRecipe( + recipe="timeseries_surface_difference_scores.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "DIFFERENCE_METHOD": difference_method, + "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + if conf.SCORES_RMSE_SPATIAL: base_model = models[0] for model, field, method in itertools.product( From 63e12f5e0094718f9175dee855e52f0cad603ccd Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 24 Jun 2026 15:55:55 +0100 Subject: [PATCH 002/117] Recipe for 20d spatial plots of scores metrics #2214 --- .../surface_difference_scores.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/CSET/recipes/verification/surface_difference_scores.yaml diff --git a/src/CSET/recipes/verification/surface_difference_scores.yaml b/src/CSET/recipes/verification/surface_difference_scores.yaml new file mode 100644 index 000000000..079fb944b --- /dev/null +++ b/src/CSET/recipes/verification/surface_difference_scores.yaml @@ -0,0 +1,46 @@ +category: Scores +title: "bmc $METHOD $VARNAME\nScores $DIFFERENCE_METHOD between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME + for all times. The RMSE is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). This recipe preserves the time, latitude and longitude coordinates. + This means a sequence of spatial plots of the RMSE can be produced. + + A larger RMSE implies a greater error than a smaller RMSE. An + RMSE of zero implies the two field match. The RMSE is calculated + on the grid point and thus a spatial view of the RMSE provides useful + information about where the differences are, or if placement errors + are domininating the score (usually indicated by dipoles). + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_difference + preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + difference_method: $DIFFERENCE_METHOD + + - operator: collapse.collapse + coordinate: ["time"] + method: $METHOD + + - operator: plot.spatial_pcolormesh_plot + + - operator: write.write_cube_to_nc + overwrite: True From 68ee9045001ae54173502b0ad0447dde5e90a171 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 24 Jun 2026 15:56:38 +0100 Subject: [PATCH 003/117] Recipe for 2-d timeseries plots of scores metrics #2214 --- .../timeseries_surface_difference_scores.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml new file mode 100644 index 000000000..d007e975f --- /dev/null +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml @@ -0,0 +1,44 @@ +category: Scores +title: "bmc $VARNAME\nScores $DIFFERENCE_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Root Mean Square Error in $VARNAME + for all times. The RMSE is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). + This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the RMSE is + collapsed over all other coordinates in the cube and calculated for every timestep. + + A larger RMSE implies a greater error than a smaller RMSE. An + RMSE of zero implies the two fields match. The RMSE is calculated + on the grid point and thus a spatial view of the RMSE provides useful + information about where the differences are, or if placement errors + are domininating the score (usually indicated by dipoles). + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_difference + preserved_coordinates: ["time"] + difference_method: $DIFFERENCE_METHOD + + - operator: plot.plot_line_series + series_coordinate: time + + - operator: write.write_cube_to_nc + overwrite: True From 4ba0e650da687fa9564c56dc11fe5fe2e4550b96 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 24 Jun 2026 16:04:34 +0100 Subject: [PATCH 004/117] Operator for scores metrics #2214 --- src/CSET/operators/scoreswrappers.py | 86 +++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 6ca1ee9bf..9badba33f 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -183,12 +183,92 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = base, other = _sort_cubes_for_verification(cubes) # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. - RMSE = xr.DataArray.to_iris( + scores_cube = xr.DataArray.to_iris( scores.continuous.rmse( xr.DataArray.from_iris(other), xr.DataArray.from_iris(base), preserve_dims=preserved_coordinates, ) ) - RMSE.rename(f"RMSE_of_{base.name()}") - return RMSE + scores_cube.rename(f"RMSE_of_{base.name()}") + return scores_cube + + +def scores_mae(cubes: CubeList, preserved_coordinates: list[str] | str | None = None): + """Calculate the Mean Absolute Error (MAE) using scores.""" + base, other = _sort_cubes_for_verification(cubes) + # Scores operators on xarray data arrays, so we transform the iris cube into an array, + # apply scores, and then transform it back. + scores_cube = xr.DataArray.to_iris( + scores.continuous.mae( + xr.DataArray.from_iris(other), + xr.DataArray.from_iris(base), + preserve_dims=preserved_coordinates, + ) + ) + scores_cube.rename(f"MAE_of_{base.name()}") + return scores_cube + + +def scores_additive_bias( + cubes: CubeList, preserved_coordinates: list[str] | str | None = None +): + """Calculate the Additive Bias (Mean Error) using scores.""" + base, other = _sort_cubes_for_verification(cubes) + # Scores operators on xarray data arrays, so we transform the iris cube into an array, + # apply scores, and then transform it back. + scores_cube = xr.DataArray.to_iris( + scores.continuous.additive_bias( + xr.DataArray.from_iris(other), + xr.DataArray.from_iris(base), + preserve_dims=preserved_coordinates, + ) + ) + scores_cube.rename(f"Additive_Bias_of_{base.name()}") + return scores_cube + + +def scores_correlation_pearsonr( + cubes: CubeList, preserved_coordinates: list[str] | str | None = None +): + """Calculate the Pearson's Correlation using scores.""" + base, other = _sort_cubes_for_verification(cubes) + # Scores operators on xarray data arrays, so we transform the iris cube into an array, + # apply scores, and then transform it back. + scores_cube = xr.DataArray.to_iris( + scores.continuous.correlation.pearsonr( + xr.DataArray.from_iris(other), + xr.DataArray.from_iris(base), + preserve_dims=preserved_coordinates, + ) + ) + scores_cube.rename(f"Pearson_Correlation_of_{base.name()}") + return scores_cube + + +def scores_difference( + cubes: CubeList, + preserved_coordinates: list[str] | str | None = None, + difference_method: str | None = None, +): + """Select which scores metric to use.""" + # initialise the output cubelist + cubes_output = iris.cube.CubeList([]) + + if difference_method == "RMSE": + # Use the RMSE method + cubes_output.append(scores_rmse(cubes, preserved_coordinates)) + + if difference_method == "additive_bias": + # Use the MAE method + cubes_output.append(scores_additive_bias(cubes, preserved_coordinates)) + + if difference_method == "MAE": + # Use the MAE method + cubes_output.append(scores_mae(cubes, preserved_coordinates)) + + if difference_method == "correlation_pearsonr": + # Use the MAE method + cubes_output.append(scores_correlation_pearsonr(cubes, preserved_coordinates)) + + return cubes_output From 722973674e2179a1b7ed262d83a3aa6f86bdb370 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 08:19:59 +0100 Subject: [PATCH 005/117] Rose GUI edits for scorer metrics #2214 --- .../meta/verification/rose-meta.conf | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 53c5cbfb5..4e6205d95 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -59,17 +59,27 @@ ns=Verification/Scores sort-key=scores title=Scores -[template variables=SCORES_RMSE_SPATIAL] +[template variables=SCORES_SPATIAL_DIFFERENCE] ns=Verification/Scores -description=Create spatially mapped RMSE plots for the specified surface fields. - The analysis method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used. -type=python_boolean +description=Create spatially mapped difference plots for the specified surface fields. + Select all spatial analysis methods required. + Option1: Root Mean Squared Difference + Option2: Mean Difference + Option3: Mean Absolute Difference + + For the time coordinate, the analysis method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used. + +type=python_boolean,python_boolean,python_boolean compulsory=true sort-key=scores1 -[template variables=SCORES_RMSE_TIMESERIES] +[template variables=SCORES_DIFFERENCE_TIMESERIES] ns=Verification/Scores -description=Create a timeseries of the RMSE for the specified surface fields. -type=python_boolean +description=Create a difference timeseries for the specified surface fields. + Select all spatial analysis methods required. + Option1: Root Mean Squared Difference + Option2: Mean Difference + Option3: Mean Absolute Difference +type=python_boolean,python_boolean,python_boolean compulsory=true sort-key=scores2 From ab7c205524339312e9f48fa3aa09535d94d7875e Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 08:20:59 +0100 Subject: [PATCH 006/117] Tidy loaders for scorer metrics #2214 --- src/CSET/loaders/verification.py | 36 +++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 46b2c04bb..9b8f0e5fb 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -26,21 +26,22 @@ def load(conf: Config): # Models are listed in order, so model 1 is the first element. if any(conf.SCORES_SPATIAL_DIFFERENCE): + """Produce 2-d spatial plots of scorers metrics.""" base_model = models[0] - difference_methods = [] + scores_methods = [] if conf.SCORES_SPATIAL_DIFFERENCE[0]: - difference_methods.append("RMSE") + scores_methods.append("RMSE") if conf.SCORES_SPATIAL_DIFFERENCE[1]: - difference_methods.append("additive_bias") + scores_methods.append("additive_bias") if conf.SCORES_SPATIAL_DIFFERENCE[2]: - difference_methods.append("MAE") + scores_methods.append("MAE") # if conf.SCORES_SPATIAL_DIFFERENCE[3]: - # difference_methods.append("correlation_pearsonr") - for model, field, method, difference_method in itertools.product( + # scores_methods.append("correlation_pearsonr") + for model, field, method, scores_method in itertools.product( models[1:], conf.SURFACE_FIELDS, conf.SPATIAL_SURFACE_FIELD_METHOD, - difference_methods, + scores_methods, ): yield RawRecipe( recipe="surface_difference_scores.yaml", @@ -49,7 +50,7 @@ def load(conf: Config): "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], "METHOD": method, - "DIFFERENCE_METHOD": difference_method, + "SCORES_METHOD": scores_method, "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA @@ -60,18 +61,19 @@ def load(conf: Config): ) if any(conf.SCORES_DIFFERENCE_TIMESERIES): + """Produce timeseries plots of scorers metrics averaged over the domain.""" base_model = models[0] - difference_methods = [] + scores_methods = [] if conf.SCORES_DIFFERENCE_TIMESERIES[0]: - difference_methods.append("RMSE") + scores_methods.append("RMSE") if conf.SCORES_DIFFERENCE_TIMESERIES[1]: - difference_methods.append("additive_bias") + scores_methods.append("additive_bias") if conf.SCORES_DIFFERENCE_TIMESERIES[2]: - difference_methods.append("MAE") - if conf.SCORES_DIFFERENCE_TIMESERIES[3]: - difference_methods.append("correlation_pearsonr") - for model, field, difference_method in itertools.product( - models[1:], conf.SURFACE_FIELDS, difference_methods + scores_methods.append("MAE") + # if conf.SCORES_DIFFERENCE_TIMESERIES[3]: + # scores_methods.append("correlation_pearsonr") + for model, field, scores_method in itertools.product( + models[1:], conf.SURFACE_FIELDS, scores_methods ): yield RawRecipe( recipe="timeseries_surface_difference_scores.yaml", @@ -79,7 +81,7 @@ def load(conf: Config): "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], - "DIFFERENCE_METHOD": difference_method, + "SCORES_METHOD": scores_method, "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA From 6100386ac3d804994124d0cb0568e1d8b74a1194 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 08:21:43 +0100 Subject: [PATCH 007/117] Tidy recipe for spatial plots of scorer metrics #2214 --- .../recipes/verification/surface_difference_scores.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores.yaml b/src/CSET/recipes/verification/surface_difference_scores.yaml index 079fb944b..ea42d483e 100644 --- a/src/CSET/recipes/verification/surface_difference_scores.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores.yaml @@ -1,5 +1,5 @@ category: Scores -title: "bmc $METHOD $VARNAME\nScores $DIFFERENCE_METHOD between $OTHER_MODEL and $BASE_MODEL" +title: "bmc $METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME for all times. The RMSE is calculated based on that used in the @@ -32,9 +32,9 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_difference + - operator: scoreswrappers.scores_metrics preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - difference_method: $DIFFERENCE_METHOD + scores_method: $SCORES_METHOD - operator: collapse.collapse coordinate: ["time"] From 81b21dc05b03580064e152274c24fa5ad529747f Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 08:22:24 +0100 Subject: [PATCH 008/117] Tidy recipe for timeseries plots of scorer metrics #2214 --- .../verification/timeseries_surface_difference_scores.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml index d007e975f..63486536e 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml @@ -1,5 +1,5 @@ category: Scores -title: "bmc $VARNAME\nScores $DIFFERENCE_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "bmc $VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error in $VARNAME for all times. The RMSE is calculated based on that used in the @@ -33,9 +33,9 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_difference + - operator: scoreswrappers.scores_metrics preserved_coordinates: ["time"] - difference_method: $DIFFERENCE_METHOD + scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From f136481efe8a6c075461f423c9f840aae1705ca1 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 08:22:56 +0100 Subject: [PATCH 009/117] Tidy operator for scorer metrics #2214 --- src/CSET/operators/scoreswrappers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 9badba33f..2166eb96a 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -162,7 +162,7 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = Returns ------- - RMSE: iris.cube.Cube + scores_cube: iris.cube.Cube A cube containing the RMSE between the base and other cube. References @@ -246,28 +246,28 @@ def scores_correlation_pearsonr( return scores_cube -def scores_difference( +def scores_metrics( cubes: CubeList, preserved_coordinates: list[str] | str | None = None, - difference_method: str | None = None, + scores_method: str | None = None, ): """Select which scores metric to use.""" # initialise the output cubelist cubes_output = iris.cube.CubeList([]) - if difference_method == "RMSE": + if scores_method == "RMSE": # Use the RMSE method cubes_output.append(scores_rmse(cubes, preserved_coordinates)) - if difference_method == "additive_bias": + if scores_method == "additive_bias": # Use the MAE method cubes_output.append(scores_additive_bias(cubes, preserved_coordinates)) - if difference_method == "MAE": + if scores_method == "MAE": # Use the MAE method cubes_output.append(scores_mae(cubes, preserved_coordinates)) - if difference_method == "correlation_pearsonr": + if scores_method == "correlation_pearsonr": # Use the MAE method cubes_output.append(scores_correlation_pearsonr(cubes, preserved_coordinates)) From 0d2a5d585d169fea246a029e41b9d8bb46024723 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 09:11:13 +0100 Subject: [PATCH 010/117] Tidy comments in operator for scorer metrics #2214 --- src/CSET/operators/scoreswrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 2166eb96a..b8d1df154 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -260,7 +260,7 @@ def scores_metrics( cubes_output.append(scores_rmse(cubes, preserved_coordinates)) if scores_method == "additive_bias": - # Use the MAE method + # Use the Additive Bias (aka mean error) method cubes_output.append(scores_additive_bias(cubes, preserved_coordinates)) if scores_method == "MAE": @@ -268,7 +268,7 @@ def scores_metrics( cubes_output.append(scores_mae(cubes, preserved_coordinates)) if scores_method == "correlation_pearsonr": - # Use the MAE method + # Use the Pearson's Correlation method cubes_output.append(scores_correlation_pearsonr(cubes, preserved_coordinates)) return cubes_output From 67dcd2d4d9e4e53839aa0116a62b4b4d6f1d85db Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 11:18:47 +0100 Subject: [PATCH 011/117] Changed the Rose GUI for the scores metrics #2214 --- .../meta/verification/rose-meta.conf | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 4e6205d95..54b93c0fe 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -53,6 +53,114 @@ sort-key=met5 ################################### + + +# Scores spatial plots +[Verification/ScoresSpatial] +ns=Verification/ScoresSpatial +sort-key=scoresspatial +title=Scores spatial + + +[template variables=SCORES_SPATIAL_RMSE] +ns=Verification/ScoresSpatial +title=Root Mean Square Error +description=SCORES COUNTINOUS METRIC. + Plots a 2-d spatial plot of the RMSE between two models. + The computation is implemented by the scores package. + For the time coordinate, the method(s) set in + SPATIAL_SURFACE_FIELD_METHOD will be used. +help=This computes the metric using the scores package. Details of this can + be found at https://scores.readthedocs.io/en/stable/included.html#continuous +type=python_boolean +compulsory=true +sort-key=sc-sp-1 + +[template variables=SCORES_SPATIAL_MAE] +ns=Verification/ScoresSpatial +title=Mean Absolute Error +description=SCORES COUNTINOUS METRIC. + Plots a 2-d spatial plot of the MAE between two models. + The computation is implemented by the scores package. + For the time coordinate, the method(s) set in + SPATIAL_SURFACE_FIELD_METHOD will be used. +help=This computes the metric using the scores package. Details of this can + be found at https://scores.readthedocs.io/en/stable/included.html#continuous +type=python_boolean +compulsory=true +sort-key=sc-sp-2 + +[template variables=SCORES_SPATIAL_AB] +ns=Verification/ScoresSpatial +title=Additive Bias (Mean Error) +description=SCORES COUNTINOUS METRIC. + Plots a 2-d spatial plot of the Additive Bias between two models. + The Additive Bias is also known as the Mean Error. + The computation is implemented by the scores package. + For the time coordinate, the method(s) set in + SPATIAL_SURFACE_FIELD_METHOD will be used. +help=This computes the metric using the scores package. Details of this can + be found at https://scores.readthedocs.io/en/stable/included.html#continuous +type=python_boolean +compulsory=true +sort-key=sc-sp-3 + +# Scores timeseries plots +[Verification/ScoresTimeseries] +ns=Verification/ScoresTimeseries +sort-key=scorestimeseries +title=Scores timeseries + +[template variables=SCORES_TIMESERIES_RMSE] +ns=Verification/ScoresTimeseries +title=Root Mean Square Error +description=SCORES COUNTINOUS METRIC. + Plots a timeseries plot of the RMSE between two models. + The computation is implemented by the scores package. + For the time coordinate, the method(s) set in + SPATIAL_SURFACE_FIELD_METHOD will be used. +help=This computes the metric using the scores package. Details of this can + be found at https://scores.readthedocs.io/en/stable/included.html#continuous +type=python_boolean +compulsory=true +sort-key=sc-ts-1 + +[template variables=SCORES_TIMESERIES_MAE] +ns=Verification/ScoresTimeseries +title=Mean Absolute Error +description=SCORES COUNTINOUS METRIC. + Plots a timeseries plot of the MAE between two models. + The computation is implemented by the scores package. + For the time coordinate, the method(s) set in + SPATIAL_SURFACE_FIELD_METHOD will be used. +help=This computes the metric using the scores package. Details of this can + be found at https://scores.readthedocs.io/en/stable/included.html#continuous +type=python_boolean +compulsory=true +sort-key=sc-ts-2 + +[template variables=SCORES_TIMESERIES_AB] +ns=Verification/ScoresTimeseries +title=Additive Bias (Mean Error) +description=SCORES COUNTINOUS METRIC. + Plots a timeseries plot of the Additive Bias between two models. + The Additive Bias is also known as the Mean Error. + The computation is implemented by the scores package. + For the time coordinate, the method(s) set in + SPATIAL_SURFACE_FIELD_METHOD will be used. +help=This computes the metric using the scores package. Details of this can + be found at https://scores.readthedocs.io/en/stable/included.html#continuous +type=python_boolean +compulsory=true +sort-key=sc-ts-3 + + + + + + + + # Scores [Verification/Scores] ns=Verification/Scores From a658473f77f6feace82aea889766dc420cd74f87 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 11:19:41 +0100 Subject: [PATCH 012/117] Changed the scores loaders to reflect the Rose GUI redesign #2214 --- src/CSET/loaders/verification.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 9b8f0e5fb..b19b6f68e 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -29,11 +29,11 @@ def load(conf: Config): """Produce 2-d spatial plots of scorers metrics.""" base_model = models[0] scores_methods = [] - if conf.SCORES_SPATIAL_DIFFERENCE[0]: + if conf.SCORES_SPATIAL_RMSE: scores_methods.append("RMSE") - if conf.SCORES_SPATIAL_DIFFERENCE[1]: + if conf.SCORES_SPATIAL_AB: scores_methods.append("additive_bias") - if conf.SCORES_SPATIAL_DIFFERENCE[2]: + if conf.SCORES_SPATIAL_MAE: scores_methods.append("MAE") # if conf.SCORES_SPATIAL_DIFFERENCE[3]: # scores_methods.append("correlation_pearsonr") @@ -64,11 +64,11 @@ def load(conf: Config): """Produce timeseries plots of scorers metrics averaged over the domain.""" base_model = models[0] scores_methods = [] - if conf.SCORES_DIFFERENCE_TIMESERIES[0]: + if conf.SCORES_TIMESERIES_RMSE: scores_methods.append("RMSE") - if conf.SCORES_DIFFERENCE_TIMESERIES[1]: + if conf.SCORES_TIMESERIES_AB: scores_methods.append("additive_bias") - if conf.SCORES_DIFFERENCE_TIMESERIES[2]: + if conf.SCORES_TIMESERIES_MAE: scores_methods.append("MAE") # if conf.SCORES_DIFFERENCE_TIMESERIES[3]: # scores_methods.append("correlation_pearsonr") From 391574d8ff51a54bbe3409a224f8dae98c5f9c9e Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 11:57:57 +0100 Subject: [PATCH 013/117] Tidied the logic in scores loaders #2214 --- src/CSET/loaders/verification.py | 58 +++++++++++++++++++------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index b19b6f68e..1e0228738 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -19,29 +19,47 @@ from CSET.recipes import Config, RawRecipe, get_models +def _get_scores_spatial_methods(conf): + """Compile of list of the required scores spatial plots.""" + scores_spatial_methods = [] + if conf.SCORES_SPATIAL_RMSE: + scores_spatial_methods.append("RMSE") + if conf.SCORES_SPATIAL_AB: + scores_spatial_methods.append("additive_bias") + if conf.SCORES_SPATIAL_MAE: + scores_spatial_methods.append("MAE") + return scores_spatial_methods + + +def _get_scores_timeseries_methods(conf): + """Compile of list of the required scores timeseries plots.""" + scores_timeseries_methods = [] + if conf.SCORES_TIMESERIES_RMSE: + scores_timeseries_methods.append("RMSE") + if conf.SCORES_TIMESERIES_AB: + scores_timeseries_methods.append("additive_bias") + if conf.SCORES_TIMESERIES_MAE: + scores_timeseries_methods.append("MAE") + # if conf.SCORES_SPATIAL_DIFFERENCE[3]: + # scores_methods.append("correlation_pearsonr") + return scores_timeseries_methods + + def load(conf: Config): """Yield recipes from the given workflow configuration.""" # Load a list of model detail dictionaries. models = get_models(conf.asdict()) # Models are listed in order, so model 1 is the first element. - if any(conf.SCORES_SPATIAL_DIFFERENCE): - """Produce 2-d spatial plots of scorers metrics.""" + scores_spatial_methods = _get_scores_spatial_methods(conf) + if scores_spatial_methods: + """Produce 2-d spatial plots of scores metrics.""" base_model = models[0] - scores_methods = [] - if conf.SCORES_SPATIAL_RMSE: - scores_methods.append("RMSE") - if conf.SCORES_SPATIAL_AB: - scores_methods.append("additive_bias") - if conf.SCORES_SPATIAL_MAE: - scores_methods.append("MAE") - # if conf.SCORES_SPATIAL_DIFFERENCE[3]: - # scores_methods.append("correlation_pearsonr") for model, field, method, scores_method in itertools.product( models[1:], conf.SURFACE_FIELDS, conf.SPATIAL_SURFACE_FIELD_METHOD, - scores_methods, + scores_spatial_methods, ): yield RawRecipe( recipe="surface_difference_scores.yaml", @@ -60,20 +78,12 @@ def load(conf: Config): aggregation=False, ) - if any(conf.SCORES_DIFFERENCE_TIMESERIES): - """Produce timeseries plots of scorers metrics averaged over the domain.""" + scores_timeseries_methods = _get_scores_timeseries_methods(conf) + if scores_timeseries_methods: + """Produce timeseries plots of scores metrics averaged over the domain.""" base_model = models[0] - scores_methods = [] - if conf.SCORES_TIMESERIES_RMSE: - scores_methods.append("RMSE") - if conf.SCORES_TIMESERIES_AB: - scores_methods.append("additive_bias") - if conf.SCORES_TIMESERIES_MAE: - scores_methods.append("MAE") - # if conf.SCORES_DIFFERENCE_TIMESERIES[3]: - # scores_methods.append("correlation_pearsonr") for model, field, scores_method in itertools.product( - models[1:], conf.SURFACE_FIELDS, scores_methods + models[1:], conf.SURFACE_FIELDS, scores_timeseries_methods ): yield RawRecipe( recipe="timeseries_surface_difference_scores.yaml", From a66c7adbd0518d8e2377d4996e873a8515152a35 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 12:23:32 +0100 Subject: [PATCH 014/117] Added scores Pearson Correlation timeseries to Rose GUI #2214 --- .../meta/verification/rose-meta.conf | 53 +++++-------------- 1 file changed, 13 insertions(+), 40 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 54b93c0fe..fa846a2d6 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -105,6 +105,9 @@ type=python_boolean compulsory=true sort-key=sc-sp-3 + +################################### + # Scores timeseries plots [Verification/ScoresTimeseries] ns=Verification/ScoresTimeseries @@ -131,8 +134,6 @@ title=Mean Absolute Error description=SCORES COUNTINOUS METRIC. Plots a timeseries plot of the MAE between two models. The computation is implemented by the scores package. - For the time coordinate, the method(s) set in - SPATIAL_SURFACE_FIELD_METHOD will be used. help=This computes the metric using the scores package. Details of this can be found at https://scores.readthedocs.io/en/stable/included.html#continuous type=python_boolean @@ -146,48 +147,20 @@ description=SCORES COUNTINOUS METRIC. Plots a timeseries plot of the Additive Bias between two models. The Additive Bias is also known as the Mean Error. The computation is implemented by the scores package. - For the time coordinate, the method(s) set in - SPATIAL_SURFACE_FIELD_METHOD will be used. help=This computes the metric using the scores package. Details of this can be found at https://scores.readthedocs.io/en/stable/included.html#continuous type=python_boolean compulsory=true sort-key=sc-ts-3 - - - - - - - -# Scores -[Verification/Scores] -ns=Verification/Scores -sort-key=scores -title=Scores - -[template variables=SCORES_SPATIAL_DIFFERENCE] -ns=Verification/Scores -description=Create spatially mapped difference plots for the specified surface fields. - Select all spatial analysis methods required. - Option1: Root Mean Squared Difference - Option2: Mean Difference - Option3: Mean Absolute Difference - - For the time coordinate, the analysis method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used. - -type=python_boolean,python_boolean,python_boolean -compulsory=true -sort-key=scores1 - -[template variables=SCORES_DIFFERENCE_TIMESERIES] -ns=Verification/Scores -description=Create a difference timeseries for the specified surface fields. - Select all spatial analysis methods required. - Option1: Root Mean Squared Difference - Option2: Mean Difference - Option3: Mean Absolute Difference -type=python_boolean,python_boolean,python_boolean +[template variables=SCORES_TIMESERIES_PC] +ns=Verification/ScoresTimeseries +title=Pearson's Correlation +description=SCORES COUNTINOUS METRIC. + Plots a timeseries plot of the Pearson's Correlation between two models. + The computation is implemented by the scores package. +help=This computes the metric using the scores package. Details of this can + be found at https://scores.readthedocs.io/en/stable/included.html#continuous +type=python_boolean compulsory=true -sort-key=scores2 +sort-key=sc-ts-4 From 91277e49e5d9b1ac7134e22d7d718ca24c04cc08 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 12:24:04 +0100 Subject: [PATCH 015/117] Added scores Pearson Correlation timeseries to loader #2214 --- src/CSET/loaders/verification.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 1e0228738..4a647fb7f 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -20,7 +20,7 @@ def _get_scores_spatial_methods(conf): - """Compile of list of the required scores spatial plots.""" + """Compile list of the required scores spatial plots.""" scores_spatial_methods = [] if conf.SCORES_SPATIAL_RMSE: scores_spatial_methods.append("RMSE") @@ -32,7 +32,7 @@ def _get_scores_spatial_methods(conf): def _get_scores_timeseries_methods(conf): - """Compile of list of the required scores timeseries plots.""" + """Compile list of the required scores timeseries plots.""" scores_timeseries_methods = [] if conf.SCORES_TIMESERIES_RMSE: scores_timeseries_methods.append("RMSE") @@ -40,8 +40,8 @@ def _get_scores_timeseries_methods(conf): scores_timeseries_methods.append("additive_bias") if conf.SCORES_TIMESERIES_MAE: scores_timeseries_methods.append("MAE") - # if conf.SCORES_SPATIAL_DIFFERENCE[3]: - # scores_methods.append("correlation_pearsonr") + if conf.SCORES_TIMESERIES_PC: + scores_timeseries_methods.append("correlation_pearsonr") return scores_timeseries_methods From f9ebf3fdf0f1ff12265c6cc8ce25a749196bd0a2 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 12:25:05 +0100 Subject: [PATCH 016/117] Removed bmc from titles of spatial plots #2214 --- src/CSET/recipes/verification/surface_difference_scores.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores.yaml b/src/CSET/recipes/verification/surface_difference_scores.yaml index ea42d483e..190f639df 100644 --- a/src/CSET/recipes/verification/surface_difference_scores.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores.yaml @@ -1,5 +1,5 @@ category: Scores -title: "bmc $METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" +title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME for all times. The RMSE is calculated based on that used in the From d7de1019f449e1260398f17f9590fc7d1fd9a1f2 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 12:25:48 +0100 Subject: [PATCH 017/117] Removed bmc from titles of timeseries plots #2214 --- .../verification/timeseries_surface_difference_scores.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml index 63486536e..29b8ab6c0 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml @@ -1,5 +1,5 @@ category: Scores -title: "bmc $VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error in $VARNAME for all times. The RMSE is calculated based on that used in the From 28865fe4ab71b011f6598a6fe1b8c400533ebee0 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 12:42:18 +0100 Subject: [PATCH 018/117] Removed legacy RMSE scores loaders #2214 --- src/CSET/loaders/verification.py | 39 -------------------------------- 1 file changed, 39 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 4a647fb7f..79c1b6b0e 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -100,42 +100,3 @@ def load(conf: Config): model_ids=[base_model["id"], model["id"]], aggregation=False, ) - - if conf.SCORES_RMSE_SPATIAL: - base_model = models[0] - for model, field, method in itertools.product( - models[1:], conf.SURFACE_FIELDS, conf.SPATIAL_SURFACE_FIELD_METHOD - ): - yield RawRecipe( - recipe="surface_rmse_scores.yaml", - variables={ - "VARNAME": field, - "BASE_MODEL": base_model["name"], - "OTHER_MODEL": model["name"], - "METHOD": method, - "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, - "SUBAREA_EXTENT": conf.SUBAREA_EXTENT - if conf.SELECT_SUBAREA - else None, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - - if conf.SCORES_RMSE_TIMESERIES: - base_model = models[0] - for model, field in itertools.product(models[1:], conf.SURFACE_FIELDS): - yield RawRecipe( - recipe="timeseries_surface_rmse_scores.yaml", - variables={ - "VARNAME": field, - "BASE_MODEL": base_model["name"], - "OTHER_MODEL": model["name"], - "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, - "SUBAREA_EXTENT": conf.SUBAREA_EXTENT - if conf.SELECT_SUBAREA - else None, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) From c171e9c5d3298c031fb6a848cf376b82438cfb14 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 14:05:08 +0100 Subject: [PATCH 019/117] Tests for the scores code #2214 --- src/CSET/tests/test_scoreswrappers.py | 72 +++++++++++ src/CSET/tests/test_verification.py | 178 ++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 src/CSET/tests/test_scoreswrappers.py create mode 100644 src/CSET/tests/test_verification.py diff --git a/src/CSET/tests/test_scoreswrappers.py b/src/CSET/tests/test_scoreswrappers.py new file mode 100644 index 000000000..78f2f3f90 --- /dev/null +++ b/src/CSET/tests/test_scoreswrappers.py @@ -0,0 +1,72 @@ +"""Unit tests for operators/scoreswrappers.py.""" + +from unittest.mock import patch + +import iris +import numpy as np +import pytest + +from CSET.operators import scoreswrappers + + +def create_cube(data, name="test", units="m"): + """Create a simple 1D Iris cube for testing.""" + return iris.cube.Cube( + np.array(data, dtype=float), + long_name=name, + units=units, + dim_coords_and_dims=[ + (iris.coords.DimCoord(np.arange(len(data)), standard_name="latitude"), 0) + ], + ) + + +class TestScoresMetrics: + """Tests for the scores_metrics function.""" + + @pytest.mark.parametrize( + "scores_method,wrapper_name", + [ + ("RMSE", "scores_rmse"), + ("MAE", "scores_mae"), + ("additive_bias", "scores_additive_bias"), + ("correlation_pearsonr", "scores_correlation_pearsonr"), + ], + ) + def test_calls_correct_scores_wrapper(self, scores_method, wrapper_name): + """The scores_metrics dispatcher should call the right wrapper.""" + cube_a = create_cube([1.0, 2.0], name="base") + cube_b = create_cube([1.5, 2.5], name="other") + cubes = iris.cube.CubeList([cube_a, cube_b]) + expected_cube = create_cube([0.0, 0.0], name="expected") + + with patch( + f"CSET.operators.scoreswrappers.{wrapper_name}", + return_value=expected_cube, + ) as mock_wrapper: + result = scoreswrappers.scores_metrics( + cubes, + preserved_coordinates=["time"], + scores_method=scores_method, + ) + + mock_wrapper.assert_called_once_with(cubes, ["time"]) + assert isinstance(result, iris.cube.CubeList) + assert len(result) == 1 + assert result[0] is expected_cube + + @pytest.mark.parametrize("scores_method", [None, "", "invalid_method"]) + def test_returns_empty_cubelist_for_unknown_method(self, scores_method): + """Unknown or missing methods should return an empty CubeList.""" + cube_a = create_cube([1.0]) + cube_b = create_cube([2.0]) + cubes = iris.cube.CubeList([cube_a, cube_b]) + + result = scoreswrappers.scores_metrics( + cubes, + preserved_coordinates=None, + scores_method=scores_method, + ) + + assert isinstance(result, iris.cube.CubeList) + assert len(result) == 0 diff --git a/src/CSET/tests/test_verification.py b/src/CSET/tests/test_verification.py new file mode 100644 index 000000000..219e1fe4f --- /dev/null +++ b/src/CSET/tests/test_verification.py @@ -0,0 +1,178 @@ +"""Unit tests for loaders/verification.py.""" + +from unittest.mock import MagicMock, patch + +from CSET.loaders.verification import ( + _get_scores_spatial_methods, + _get_scores_timeseries_methods, + load, +) +from CSET.recipes import Config, RawRecipe + + +class TestGetScoresSpatialMethods: + """Test the _get_scores_spatial_methods helper.""" + + def make_conf(self, **kwargs): + """Test make_conf.""" + conf = MagicMock(spec=Config) + conf.SCORES_SPATIAL_RMSE = kwargs.get("SCORES_SPATIAL_RMSE", False) + conf.SCORES_SPATIAL_AB = kwargs.get("SCORES_SPATIAL_AB", False) + conf.SCORES_SPATIAL_MAE = kwargs.get("SCORES_SPATIAL_MAE", False) + return conf + + def test_returns_empty_list_when_no_spatial_scores_enabled(self): + """Test some code.""" + conf = self.make_conf() + + assert _get_scores_spatial_methods(conf) == [] + + def test_returns_expected_spatial_score_methods(self): + """Test some code.""" + conf = self.make_conf( + SCORES_SPATIAL_RMSE=True, + SCORES_SPATIAL_AB=True, + SCORES_SPATIAL_MAE=True, + ) + + assert _get_scores_spatial_methods(conf) == [ + "RMSE", + "additive_bias", + "MAE", + ] + + +class TestGetScoresTimeseriesMethods: + """Test the _get_scores_timeseries_methods helper.""" + + def make_conf(self, **kwargs): + """Test some code.""" + conf = MagicMock(spec=Config) + conf.SCORES_TIMESERIES_RMSE = kwargs.get("SCORES_TIMESERIES_RMSE", False) + conf.SCORES_TIMESERIES_AB = kwargs.get("SCORES_TIMESERIES_AB", False) + conf.SCORES_TIMESERIES_MAE = kwargs.get("SCORES_TIMESERIES_MAE", False) + conf.SCORES_TIMESERIES_PC = kwargs.get("SCORES_TIMESERIES_PC", False) + return conf + + def test_returns_empty_list_when_no_timeseries_scores_enabled(self): + """Test some code.""" + conf = self.make_conf() + + assert _get_scores_timeseries_methods(conf) == [] + + def test_returns_expected_timeseries_score_methods(self): + """Test some code.""" + conf = self.make_conf( + SCORES_TIMESERIES_RMSE=True, + SCORES_TIMESERIES_AB=True, + SCORES_TIMESERIES_MAE=True, + SCORES_TIMESERIES_PC=True, + ) + + assert _get_scores_timeseries_methods(conf) == [ + "RMSE", + "additive_bias", + "MAE", + "correlation_pearsonr", + ] + + +class TestLoadVerificationRecipes: + """Test the load function in loaders/verification.py.""" + + def make_conf(self, **kwargs): + """Test some code.""" + conf = MagicMock(spec=Config) + conf.SCORES_SPATIAL_RMSE = kwargs.get("SCORES_SPATIAL_RMSE", False) + conf.SCORES_SPATIAL_AB = kwargs.get("SCORES_SPATIAL_AB", False) + conf.SCORES_SPATIAL_MAE = kwargs.get("SCORES_SPATIAL_MAE", False) + conf.SCORES_TIMESERIES_RMSE = kwargs.get("SCORES_TIMESERIES_RMSE", False) + conf.SCORES_TIMESERIES_AB = kwargs.get("SCORES_TIMESERIES_AB", False) + conf.SCORES_TIMESERIES_MAE = kwargs.get("SCORES_TIMESERIES_MAE", False) + conf.SCORES_TIMESERIES_PC = kwargs.get("SCORES_TIMESERIES_PC", False) + conf.SURFACE_FIELDS = kwargs.get("SURFACE_FIELDS", ["temperature"]) + conf.SPATIAL_SURFACE_FIELD_METHOD = kwargs.get( + "SPATIAL_SURFACE_FIELD_METHOD", ["SEQ"] + ) + conf.SELECT_SUBAREA = kwargs.get("SELECT_SUBAREA", False) + conf.SUBAREA_TYPE = kwargs.get("SUBAREA_TYPE", None) + conf.SUBAREA_EXTENT = kwargs.get("SUBAREA_EXTENT", None) + conf.asdict.return_value = kwargs.get("asdict", {}) + return conf + + @patch("CSET.loaders.verification.get_models") + def test_load_yields_no_recipes_when_no_scores_selected(self, mock_get_models): + """Test some code.""" + mock_get_models.return_value = [ + {"name": "base", "id": 1}, + {"name": "other", "id": 2}, + ] + conf = self.make_conf() + + assert list(load(conf)) == [] + mock_get_models.assert_called_once_with(conf.asdict.return_value) + + @patch("CSET.loaders.verification.get_models") + def test_load_yields_spatial_score_recipes(self, mock_get_models): + """Test some code.""" + mock_get_models.return_value = [ + {"name": "base_model", "id": 10}, + {"name": "other_model", "id": 20}, + ] + conf = self.make_conf( + SCORES_SPATIAL_RMSE=True, + SCORES_SPATIAL_MAE=True, + SURFACE_FIELDS=["v1", "v2"], + SPATIAL_SURFACE_FIELD_METHOD=["SEQ", "AVE"], + SELECT_SUBAREA=True, + SUBAREA_TYPE="UK", + SUBAREA_EXTENT=[0, 1, 2, 3], + asdict={"dummy": "value"}, + ) + + recipes = list(load(conf)) + + assert len(recipes) == 8 + for recipe in recipes: + assert isinstance(recipe, RawRecipe) + assert recipe.recipe == "surface_difference_scores.yaml" + assert recipe.variables["BASE_MODEL"] == "base_model" + assert recipe.variables["OTHER_MODEL"] == "other_model" + assert recipe.variables["SUBAREA_TYPE"] == "UK" + assert recipe.variables["SUBAREA_EXTENT"] == [0, 1, 2, 3] + assert recipe.model_ids == [10, 20] + + mock_get_models.assert_called_once_with(conf.asdict.return_value) + + @patch("CSET.loaders.verification.get_models") + def test_load_yields_timeseries_score_recipes(self, mock_get_models): + """Test some code.""" + mock_get_models.return_value = [ + {"name": "reference", "id": 1}, + {"name": "comparison", "id": 2}, + ] + conf = self.make_conf( + SCORES_TIMESERIES_RMSE=True, + SCORES_TIMESERIES_PC=True, + SURFACE_FIELDS=["t2m"], + SELECT_SUBAREA=False, + asdict={"dummy": "value"}, + ) + + recipes = list(load(conf)) + + assert len(recipes) == 2 + assert {recipe.recipe for recipe in recipes} == { + "timeseries_surface_difference_scores.yaml" + } + + for recipe in recipes: + assert isinstance(recipe, RawRecipe) + assert recipe.variables["BASE_MODEL"] == "reference" + assert recipe.variables["OTHER_MODEL"] == "comparison" + assert recipe.variables["SUBAREA_TYPE"] is None + assert recipe.variables["SUBAREA_EXTENT"] is None + assert recipe.model_ids == [1, 2] + + score_methods = {recipe.variables["SCORES_METHOD"] for recipe in recipes} + assert score_methods == {"RMSE", "correlation_pearsonr"} From bb5ba3f8db7e7133e41dc7e6819c617923f85cd4 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 25 Jun 2026 14:28:48 +0100 Subject: [PATCH 020/117] Updated rose-suite.conf.example with the scores variables #2214 --- src/CSET/cset_workflow/rose-suite.conf.example | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 984415ba6..aac33ee0d 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -125,8 +125,14 @@ PROFILE_PLEVEL_AGGREGATION=False,False,False,False RAIN_PRESENCE_DOMAIN_MEAN_TIMESERIES=False RAIN_PRESENCE_SPATIAL_DIFFERENCE=False RAIN_PRESENCE_SPATIAL_PLOT=False -SCORES_RMSE_SPATIAL=False -SCORES_RMSE_TIMESERIES=False +!!SCORES_RMSE_SPATIAL=False +!!SCORES_RMSE_TIMESERIES=False +SCORES_SPATIAL_AB=False +SCORES_SPATIAL_MAE=False +SCORES_SPATIAL_RMSE=False +SCORES_TIMESERIES_AB=False +SCORES_TIMESERIES_MAE=False +SCORES_TIMESERIES_RMSE=False SCREEN_LEVEL_TEMPERATURE_PROBABILITIES=False !!SCREEN_LEVEL_TEMPERATURE_SPATIAL_PROBABILITY_WITHOUT_CONTROL_MEMBER=False SELECT_SUBAREA=False From 0a3c96f7ff34a7963f5268fba5ee246d34138f43 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 09:21:25 +0100 Subject: [PATCH 021/117] Removing redundant variables from rose-suite.conf.example #2214 --- src/CSET/cset_workflow/rose-suite.conf.example | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index aac33ee0d..9ce9b2736 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -125,8 +125,6 @@ PROFILE_PLEVEL_AGGREGATION=False,False,False,False RAIN_PRESENCE_DOMAIN_MEAN_TIMESERIES=False RAIN_PRESENCE_SPATIAL_DIFFERENCE=False RAIN_PRESENCE_SPATIAL_PLOT=False -!!SCORES_RMSE_SPATIAL=False -!!SCORES_RMSE_TIMESERIES=False SCORES_SPATIAL_AB=False SCORES_SPATIAL_MAE=False SCORES_SPATIAL_RMSE=False From 9358351df46f7e9a67321fc6204de0a404c547d8 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Mon, 29 Jun 2026 10:55:43 +0100 Subject: [PATCH 022/117] Apply suggestion from @jfrost-mo Applying suggestion from James F Co-authored-by: James Frost --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 80f4fc26c..3392830b0 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -54,7 +54,6 @@ sort-key=met5 ################################### - [template variables=SCORES_ALL] ns=Verification description=If selected, all scores shown below will be calculated. From 3f4256aa3c44cdff8f28a79cce4312ff0e784615 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Mon, 29 Jun 2026 11:49:44 +0100 Subject: [PATCH 023/117] Apply suggestion from @jfrost-mo Apply stylistic suggestion from James F Co-authored-by: James Frost --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 3392830b0..424e556da 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -81,7 +81,7 @@ title=Scores spatial ns=Verification/ScoresSpatial title=Root Mean Square Error description=SCORES COUNTINOUS METRIC. - Plots a 2-d spatial plot of the RMSE between two models. + Plots a 2D spatial plot of the RMSE between two models. The computation is implemented by the scores package. For the time coordinate, the method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used. From 8894a87fafdc22785cc0fd7ef9376a92dbe31c2a Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Mon, 29 Jun 2026 11:50:37 +0100 Subject: [PATCH 024/117] Apply suggestion from @jfrost-mo Apply stylistic suggestion from James F Co-authored-by: James Frost --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 424e556da..54c45b92e 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -95,7 +95,7 @@ sort-key=sc-sp-1 ns=Verification/ScoresSpatial title=Mean Absolute Error description=SCORES COUNTINOUS METRIC. - Plots a 2-d spatial plot of the MAE between two models. + Plots a 2D spatial plot of the MAE between two models. The computation is implemented by the scores package. For the time coordinate, the method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used. From f3899db1a107cbc7f70454550d45d956320d22ca Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Mon, 29 Jun 2026 11:52:16 +0100 Subject: [PATCH 025/117] Apply suggestion from @jfrost-mo Apply stylistic suggestion from James F Co-authored-by: James Frost --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 54c45b92e..552df6070 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -109,7 +109,7 @@ sort-key=sc-sp-2 ns=Verification/ScoresSpatial title=Additive Bias (Mean Error) description=SCORES COUNTINOUS METRIC. - Plots a 2-d spatial plot of the Additive Bias between two models. + Plots a 2D spatial plot of the Additive Bias between two models. The Additive Bias is also known as the Mean Error. The computation is implemented by the scores package. For the time coordinate, the method(s) set in From e4a76508b44585dd59b993dd24066916ade356c6 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Mon, 29 Jun 2026 11:53:16 +0100 Subject: [PATCH 026/117] Apply suggestion from @jfrost-mo Applying comment change suggested by James F Co-authored-by: James Frost --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 552df6070..b2e5c6de1 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -172,7 +172,7 @@ sort-key=sc-ts-3 ns=Verification/ScoresTimeseries title=Pearson's Correlation description=SCORES COUNTINOUS METRIC. - Plots a timeseries plot of the Pearson's Correlation between two models. + Plots a timeseries plot of the Pearson’s Correlation Coefficient between two models. The computation is implemented by the scores package. help=This computes the metric using the scores package. Details of this can be found at https://scores.readthedocs.io/en/stable/included.html#continuous From 11445b8fb730a9e8e67f019891d42a3925ef9ad2 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Mon, 29 Jun 2026 11:58:42 +0100 Subject: [PATCH 027/117] Apply suggestion from @jfrost-mo Apply comment suggestion from James F Co-authored-by: James Frost --- src/CSET/loaders/verification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index de1398faf..1c927ebbd 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -53,7 +53,7 @@ def load(conf: Config): scores_spatial_methods = _get_scores_spatial_methods(conf) if scores_spatial_methods: - """Produce 2-d spatial plots of scores metrics.""" + # Produce 2D spatial plots of scores metrics. base_model = models[0] for model, field, method, scores_method in itertools.product( models[1:], From 6722245b854985449b982505f1c12f1abdda0018 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Mon, 29 Jun 2026 12:21:00 +0100 Subject: [PATCH 028/117] Apply suggestion from @jfrost-mo Apply comment suggestion from James F Co-authored-by: James Frost --- src/CSET/recipes/verification/surface_difference_scores.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores.yaml b/src/CSET/recipes/verification/surface_difference_scores.yaml index 190f639df..2839e083c 100644 --- a/src/CSET/recipes/verification/surface_difference_scores.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores.yaml @@ -7,7 +7,7 @@ description: | This means a sequence of spatial plots of the RMSE can be produced. A larger RMSE implies a greater error than a smaller RMSE. An - RMSE of zero implies the two field match. The RMSE is calculated + RMSE of zero indicates the two field match. The RMSE is calculated on the grid point and thus a spatial view of the RMSE provides useful information about where the differences are, or if placement errors are domininating the score (usually indicated by dipoles). From 68898796531b681e915f74deb165610ce2e6e7fd Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 12:01:54 +0100 Subject: [PATCH 029/117] Applying change to from docstring to regular comment #2214 --- src/CSET/loaders/verification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 1c927ebbd..cd979e259 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -80,7 +80,7 @@ def load(conf: Config): scores_timeseries_methods = _get_scores_timeseries_methods(conf) if scores_timeseries_methods: - """Produce timeseries plots of scores metrics averaged over the domain.""" + # Produce timeseries plots of scores metrics averaged over the domain. base_model = models[0] for model, field, scores_method in itertools.product( models[1:], conf.SURFACE_FIELDS, scores_timeseries_methods From 8d09a91fd2447fa4b4e18a3654a2192646980b32 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 12:13:27 +0100 Subject: [PATCH 030/117] Redrafted to use match statment to select the scores method #2214 --- src/CSET/operators/scoreswrappers.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index b8d1df154..2c69b4fcf 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -255,20 +255,24 @@ def scores_metrics( # initialise the output cubelist cubes_output = iris.cube.CubeList([]) - if scores_method == "RMSE": - # Use the RMSE method - cubes_output.append(scores_rmse(cubes, preserved_coordinates)) + match scores_method: + case "RMSE": + # Use the RMSE method + return scores_rmse(cubes, preserved_coordinates) - if scores_method == "additive_bias": - # Use the Additive Bias (aka mean error) method - cubes_output.append(scores_additive_bias(cubes, preserved_coordinates)) + case "additive_bias": + # Use the Additive Bias (aka mean error) method + return scores_additive_bias(cubes, preserved_coordinates) - if scores_method == "MAE": - # Use the MAE method - cubes_output.append(scores_mae(cubes, preserved_coordinates)) + case "MAE": + # Use the MAE method + return scores_mae(cubes, preserved_coordinates) - if scores_method == "correlation_pearsonr": - # Use the Pearson's Correlation method - cubes_output.append(scores_correlation_pearsonr(cubes, preserved_coordinates)) + case "correlation_pearsonr": + # Use the Pearson's Correlation method + return scores_correlation_pearsonr(cubes, preserved_coordinates) + + case _: + raise ValueError(f"No such scores metric: {scores_method}") return cubes_output From 1eab7698328885d722ce4156beb89e6776a5f9ad Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 12:28:01 +0100 Subject: [PATCH 031/117] Re-committing rose-meta.conf #2214 --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index b2e5c6de1..2ce0631d9 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -133,7 +133,7 @@ title=Scores timeseries ns=Verification/ScoresTimeseries title=Root Mean Square Error description=SCORES COUNTINOUS METRIC. - Plots a timeseries plot of the RMSE between two models. + Plot a timeseries plot of the RMSE between two models. The computation is implemented by the scores package. For the time coordinate, the method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used. From b7bb8e1697ac1c1dc6943183923bfa96e5ddd552 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 12:34:42 +0100 Subject: [PATCH 032/117] Minor comment change #2214 --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 2ce0631d9..b2e5c6de1 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -133,7 +133,7 @@ title=Scores timeseries ns=Verification/ScoresTimeseries title=Root Mean Square Error description=SCORES COUNTINOUS METRIC. - Plot a timeseries plot of the RMSE between two models. + Plots a timeseries plot of the RMSE between two models. The computation is implemented by the scores package. For the time coordinate, the method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used. From 5282d67c33cc348162d7bbc5a156a0d7156e8e2c Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:20:46 +0100 Subject: [PATCH 033/117] Modified loaders for scores metrics #2214 --- src/CSET/loaders/verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index cd979e259..3a6a730ac 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -62,7 +62,7 @@ def load(conf: Config): scores_spatial_methods, ): yield RawRecipe( - recipe="surface_difference_scores.yaml", + recipe=f"surface_difference_scores_{scores_method}.yaml", variables={ "VARNAME": field, "BASE_MODEL": base_model["name"], @@ -86,7 +86,7 @@ def load(conf: Config): models[1:], conf.SURFACE_FIELDS, scores_timeseries_methods ): yield RawRecipe( - recipe="timeseries_surface_difference_scores.yaml", + recipe=f"timeseries_surface_difference_scores_{scores_method}.yaml", variables={ "VARNAME": field, "BASE_MODEL": base_model["name"], From 9b90b0aad9f285ee1076dd224b4ff55967f9fc86 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:21:55 +0100 Subject: [PATCH 034/117] Recipe for spatial scores RMSE #2214 --- .../surface_difference_scores_RMSE.yaml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml diff --git a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml new file mode 100644 index 000000000..c4459f77c --- /dev/null +++ b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml @@ -0,0 +1,47 @@ +category: Scores +title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME + for all times. The RMSE is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). + This recipe preserves the time, latitude and longitude coordinates. + This means a sequence of spatial plots of the RMSE can be produced. + + A larger RMSE implies a greater error than a smaller RMSE. An + RMSE of zero indicates the two fields match. The RMSE is calculated + on the grid point and thus a spatial view of the RMSE provides useful + information about where the differences are, or if placement errors + are domininating the score (usually indicated by dipoles). + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_metrics + preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + scores_method: $SCORES_METHOD + + - operator: collapse.collapse + coordinate: ["time"] + method: $METHOD + + - operator: plot.spatial_pcolormesh_plot + + - operator: write.write_cube_to_nc + overwrite: True From 1b35d182766cefa9823cab4ef804ca877f2fccec Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:22:16 +0100 Subject: [PATCH 035/117] Recipe for spatial scores MAE #2214 --- .../surface_difference_scores_MAE.yaml | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 src/CSET/recipes/verification/surface_difference_scores_MAE.yaml diff --git a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml new file mode 100644 index 000000000..f78fd7bd8 --- /dev/null +++ b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml @@ -0,0 +1,48 @@ +category: Scores +title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Mean Absolute Error in $METHOD of $VARNAME + for all times. The MAE is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mae). + This recipe preserves the time, latitude and longitude coordinates. + This means a sequence of spatial plots of the MAE can be produced. + + A larger MAE implies a greater error than a smaller MAE. An + RMSE of zero indicates the two fields most likely match. The MAE is calculated + on the grid point and thus a spatial view of the MAE provides useful + information about where the differences are, or if placement errors + are domininating the score (usually indicated by dipoles). MAE is a fair + measure of error when forecasting the median and is not sensitive to outliers. + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_metrics + preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + scores_method: $SCORES_METHOD + + - operator: collapse.collapse + coordinate: ["time"] + method: $METHOD + + - operator: plot.spatial_pcolormesh_plot + + - operator: write.write_cube_to_nc + overwrite: True From 52238e5595388d1d7fbc5e705835f3f82693f961 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:22:50 +0100 Subject: [PATCH 036/117] Recipe for spatial scores additive bias #2214 --- ...rface_difference_scores_additive_bias.yaml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml diff --git a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml new file mode 100644 index 000000000..6b0ae035e --- /dev/null +++ b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml @@ -0,0 +1,46 @@ +category: Scores +title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Mean Error (also known as the Additive Bias) in $METHOD of $VARNAME + for all times. The ME is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mean_error). + This recipe preserves the time, latitude and longitude coordinates. + This means a sequence of spatial plots of the ME can be produced. + + A larger ME implies a greater error than a smaller ME. Although a near zero ME could be indicative + of a good match between two fields, this could also arise if there are large compensating + positive and negative errors. The ME is calculated on the grid point and thus a spatial view of + the ME provides useful information about whether a field has a bias for over or under-forecasting. + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_metrics + preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + scores_method: $SCORES_METHOD + + - operator: collapse.collapse + coordinate: ["time"] + method: $METHOD + + - operator: plot.spatial_pcolormesh_plot + + - operator: write.write_cube_to_nc + overwrite: True From f9a9c94c7c6d736c7b435901726ae9305422c493 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:25:32 +0100 Subject: [PATCH 037/117] Recipe for timeseries scores pearsons correlation #2214 --- ...ifference_scores_correlation_pearsonr.yaml | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml new file mode 100644 index 000000000..0253ffc53 --- /dev/null +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml @@ -0,0 +1,42 @@ +category: Scores +title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Pearson's Correlation coefficient in $VARNAME + for all times. The PC is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.correlation.pearsonr). + This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the PC is + collapsed over all other coordinates in the cube and calculated for every timestep. + + The PC coefficient provides information on the linear relationship between two fields. + Perfectly correlated fields would yield a PC coefficient of 1, whereas perfectly anti-correlated + fields would yield a PC coefficient of -1. A value of 0 would indicate that the fields are non-correlated. + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_metrics + preserved_coordinates: ["time"] + scores_method: $SCORES_METHOD + + - operator: plot.plot_line_series + series_coordinate: time + + - operator: write.write_cube_to_nc + overwrite: True From f92575d27def32e29b188b5e4001719d92e30d22 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:26:05 +0100 Subject: [PATCH 038/117] Recipe for timeseries scores additive bias #2214 --- ...rface_difference_scores_additive_bias.yaml | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml new file mode 100644 index 000000000..1e7d3c9bc --- /dev/null +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml @@ -0,0 +1,43 @@ +category: Scores +title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Mean Error (also known as the Additive Bias) in $VARNAME + for all times. The ME is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mean_error). + This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the ME is + collapsed over all other coordinates in the cube and calculated for every timestep. + + A larger ME implies a greater error than a smaller ME. Although a near zero ME could be indicative + of a good match between two fields, this could also arise if there are large compensating positive and + negative errors. The ME is calculated on the grid point and thus a spatial view of the ME provides useful + information about whether a field has a bias for over or under-forecasting. + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_metrics + preserved_coordinates: ["time"] + scores_method: $SCORES_METHOD + + - operator: plot.plot_line_series + series_coordinate: time + + - operator: write.write_cube_to_nc + overwrite: True From c6576053061d83d131c585292f994cf50513cfea Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:26:23 +0100 Subject: [PATCH 039/117] Recipe for timeseries scores MAE #2214 --- ...eseries_surface_difference_scores_MAE.yaml | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml new file mode 100644 index 000000000..f2bfb6e46 --- /dev/null +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml @@ -0,0 +1,45 @@ +category: Scores +title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Mean Absolute Error in $VARNAME + for all times. The MAE is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mae). + This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the MAE is + collapsed over all other coordinates in the cube and calculated for every timestep. + + A larger MAE implies a greater error than a smaller MAE. An + RMSE of zero indicates the two fields most likely match. The MAE is calculated + on the grid point and thus a spatial view of the MAE provides useful + information about where the differences are, or if placement errors + are domininating the score (usually indicated by dipoles). MAE is a fair + measure of error when forecasting the median and is not sensitive to outliers. + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_metrics + preserved_coordinates: ["time"] + scores_method: $SCORES_METHOD + + - operator: plot.plot_line_series + series_coordinate: time + + - operator: write.write_cube_to_nc + overwrite: True From cb7160a630d650849249a290d5aac807ee738ad7 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:26:38 +0100 Subject: [PATCH 040/117] Recipe for timeseries scores RMSE #2214 --- ...series_surface_difference_scores_RMSE.yaml | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml new file mode 100644 index 000000000..29b8ab6c0 --- /dev/null +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml @@ -0,0 +1,44 @@ +category: Scores +title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Root Mean Square Error in $VARNAME + for all times. The RMSE is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). + This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the RMSE is + collapsed over all other coordinates in the cube and calculated for every timestep. + + A larger RMSE implies a greater error than a smaller RMSE. An + RMSE of zero implies the two fields match. The RMSE is calculated + on the grid point and thus a spatial view of the RMSE provides useful + information about where the differences are, or if placement errors + are domininating the score (usually indicated by dipoles). + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_metrics + preserved_coordinates: ["time"] + scores_method: $SCORES_METHOD + + - operator: plot.plot_line_series + series_coordinate: time + + - operator: write.write_cube_to_nc + overwrite: True From c9485c4be0e2fab4e0d567bed33dbae76146039c Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 29 Jun 2026 14:34:35 +0100 Subject: [PATCH 041/117] Minor change to a recipe description #2214 --- .../verification/timeseries_surface_difference_scores_RMSE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml index 29b8ab6c0..318551f7d 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml @@ -8,7 +8,7 @@ description: | collapsed over all other coordinates in the cube and calculated for every timestep. A larger RMSE implies a greater error than a smaller RMSE. An - RMSE of zero implies the two fields match. The RMSE is calculated + RMSE of zero indicates the two fields match. The RMSE is calculated on the grid point and thus a spatial view of the RMSE provides useful information about where the differences are, or if placement errors are domininating the score (usually indicated by dipoles). From 63a1e9291603a6c83892afba8c6c726d4604fd10 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 30 Jun 2026 11:25:21 +0100 Subject: [PATCH 042/117] Unit test for MAE in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index c886f87a0..5d5128967 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -28,6 +28,23 @@ from CSET.operators import scoreswrappers +def test_scores_mae(cube: iris.cube.Cube): + """Test taking the rmse between two cubes.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Take difference. + mae_cube = scoreswrappers.scores_mae(cubes) + + # As both cubes use the same data, check the mae is zero. + assert isinstance(mae_cube, iris.cube.Cube) + assert np.allclose(mae_cube.data, np.zeros_like(mae_cube.data), atol=1e-9) + assert mae_cube.standard_name is None + assert mae_cube.long_name == "MAE_of_air_temperature" + + def test_scores_rmse(cube: iris.cube.Cube): """Test taking the rmse between two cubes.""" # Data preparation. From 16c585ef7e53afd10f9c1ebe30d3d23743aead18 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 30 Jun 2026 11:31:25 +0100 Subject: [PATCH 043/117] Unit test for additive bias in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 5d5128967..9bbc22d2e 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -28,8 +28,27 @@ from CSET.operators import scoreswrappers +def test_scores_additive_bias(cube: iris.cube.Cube): + """Test taking the additive bias between two cubes.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Take difference. + additive_bias_cube = scoreswrappers.scores_additive_bias(cubes) + + # As both cubes use the same data, check the additive bias is zero. + assert isinstance(additive_bias_cube, iris.cube.Cube) + assert np.allclose( + additive_bias_cube.data, np.zeros_like(additive_bias_cube.data), atol=1e-9 + ) + assert additive_bias_cube.standard_name is None + assert additive_bias_cube.long_name == "additive_bias_of_air_temperature" + + def test_scores_mae(cube: iris.cube.Cube): - """Test taking the rmse between two cubes.""" + """Test taking the mae between two cubes.""" # Data preparation. other_cube = cube.copy() del other_cube.attributes["cset_comparison_base"] From 8175fdd8bd1e09149b668127d19093f87defca89 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 30 Jun 2026 11:37:26 +0100 Subject: [PATCH 044/117] Unit test for Pearson correlation in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 9bbc22d2e..d792752c1 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -28,6 +28,29 @@ from CSET.operators import scoreswrappers +def test_scores_correlation_pearsonr(cube: iris.cube.Cube): + """Test taking the Pearson correlation between two cubes.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Take difference. + correlation_pearsonr_cube = scoreswrappers.scores_correlation_pearsonr(cubes) + + # As both cubes use the same data, check the Pearson correlation is one. + assert isinstance(correlation_pearsonr_cube, iris.cube.Cube) + assert np.allclose( + correlation_pearsonr_cube.data - 1.0, + np.zeros_like(correlation_pearsonr_cube.data), + atol=1e-9, + ) + assert correlation_pearsonr_cube.standard_name is None + assert ( + correlation_pearsonr_cube.long_name == "correlation_pearsonr_of_air_temperature" + ) + + def test_scores_additive_bias(cube: iris.cube.Cube): """Test taking the additive bias between two cubes.""" # Data preparation. From 6cb2e245c617f80048b64979e14f1e563afde79c Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 1 Jul 2026 14:18:14 +0100 Subject: [PATCH 045/117] Unit test for MAE scores_method in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index d792752c1..9c8b417a5 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -87,6 +87,27 @@ def test_scores_mae(cube: iris.cube.Cube): assert mae_cube.long_name == "MAE_of_air_temperature" +def test_scores_metrics_mae(cube: iris.cube.Cube): + """Test taking the mae between two cubes using the metric selector.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Take difference. + mae_cube = scoreswrappers.scores_metrics( + cubes, + preserved_coordinates=["time", "grid_latitude", "grid_longitude"], + scores_method="MAE", + ) + + # As both cubes use the same data, check the mae is zero. + assert isinstance(mae_cube, iris.cube.Cube) + assert np.allclose(mae_cube.data, np.zeros_like(mae_cube.data), atol=1e-9) + assert mae_cube.standard_name is None + assert mae_cube.long_name == "MAE_of_air_temperature" + + def test_scores_rmse(cube: iris.cube.Cube): """Test taking the rmse between two cubes.""" # Data preparation. From 1604e4bab7ac80868ad14945ed211cb6a6666778 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 1 Jul 2026 14:23:25 +0100 Subject: [PATCH 046/117] Unit test for RMSE scores_method in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 9c8b417a5..599fe3d7f 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -125,6 +125,27 @@ def test_scores_rmse(cube: iris.cube.Cube): assert rmse_cube.long_name == "RMSE_of_air_temperature" +def test_scores_metrics_rmse(cube: iris.cube.Cube): + """Test taking the rmse between two cubes.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Take difference. + rmse_cube = scoreswrappers.scores_metrics( + cubes, + preserved_coordinates=["time", "grid_latitude", "grid_longitude"], + scores_method="RMSE", + ) + + # As both cubes use the same data, check the rmse is zero. + assert isinstance(rmse_cube, iris.cube.Cube) + assert np.allclose(rmse_cube.data, np.zeros_like(rmse_cube.data), atol=1e-9) + assert rmse_cube.standard_name is None + assert rmse_cube.long_name == "RMSE_of_air_temperature" + + def test_scores_rmse_nonzero(cube: iris.cube.Cube): """Test taking the rmse between two different cubes.""" # Data preparation. From 1e6c227a53ccb2fccab43c5da531bdd5272265a9 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 1 Jul 2026 14:28:07 +0100 Subject: [PATCH 047/117] Unit test for additive bias scores_method in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 599fe3d7f..5639d6b5c 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -70,6 +70,30 @@ def test_scores_additive_bias(cube: iris.cube.Cube): assert additive_bias_cube.long_name == "additive_bias_of_air_temperature" +def test_scores_metrics_additive_bias(cube: iris.cube.Cube): + """Test taking the additive bias between two cubes.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Take difference. + additive_bias_cube = scoreswrappers.scores_metrics( + cubes, + preserved_coordinates=["time", "grid_latitude", "grid_longitude"], + scores_method="additive_bias", + ) + additive_bias_cube = scoreswrappers.scores_additive_bias(cubes) + + # As both cubes use the same data, check the additive bias is zero. + assert isinstance(additive_bias_cube, iris.cube.Cube) + assert np.allclose( + additive_bias_cube.data, np.zeros_like(additive_bias_cube.data), atol=1e-9 + ) + assert additive_bias_cube.standard_name is None + assert additive_bias_cube.long_name == "additive_bias_of_air_temperature" + + def test_scores_mae(cube: iris.cube.Cube): """Test taking the mae between two cubes.""" # Data preparation. From 783cb32918355decedd2b3c0312a510a31d415f5 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 1 Jul 2026 14:31:00 +0100 Subject: [PATCH 048/117] Unit test for Pearson correlation scores_method in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 5639d6b5c..ee31b34ec 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -51,6 +51,34 @@ def test_scores_correlation_pearsonr(cube: iris.cube.Cube): ) +def test_scores_metrics_correlation_pearsonr(cube: iris.cube.Cube): + """Test taking the Pearson correlation between two cubes.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Take difference. + correlation_pearsonr_cube = scoreswrappers.scores_metrics( + cubes, + preserved_coordinates=["time"], + scores_method="correlation_pearsonr", + ) + correlation_pearsonr_cube = scoreswrappers.scores_correlation_pearsonr(cubes) + + # As both cubes use the same data, check the Pearson correlation is one. + assert isinstance(correlation_pearsonr_cube, iris.cube.Cube) + assert np.allclose( + correlation_pearsonr_cube.data - 1.0, + np.zeros_like(correlation_pearsonr_cube.data), + atol=1e-9, + ) + assert correlation_pearsonr_cube.standard_name is None + assert ( + correlation_pearsonr_cube.long_name == "correlation_pearsonr_of_air_temperature" + ) + + def test_scores_additive_bias(cube: iris.cube.Cube): """Test taking the additive bias between two cubes.""" # Data preparation. From 92c166e7c83565cc3fc6e0bca8a5515984f81599 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 1 Jul 2026 14:40:34 +0100 Subject: [PATCH 049/117] Unit test for unknown metric scores_method in scoreswrapper.py #2214 --- tests/operators/test_scoreswrappers.py | 27 +++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index ee31b34ec..410b84cd5 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -52,7 +52,7 @@ def test_scores_correlation_pearsonr(cube: iris.cube.Cube): def test_scores_metrics_correlation_pearsonr(cube: iris.cube.Cube): - """Test taking the Pearson correlation between two cubes.""" + """Test taking the Pearson correlation between two cubes using the metric selector.""" # Data preparation. other_cube = cube.copy() del other_cube.attributes["cset_comparison_base"] @@ -99,7 +99,7 @@ def test_scores_additive_bias(cube: iris.cube.Cube): def test_scores_metrics_additive_bias(cube: iris.cube.Cube): - """Test taking the additive bias between two cubes.""" + """Test taking the additive bias between two cubes using the metric selector.""" # Data preparation. other_cube = cube.copy() del other_cube.attributes["cset_comparison_base"] @@ -178,7 +178,7 @@ def test_scores_rmse(cube: iris.cube.Cube): def test_scores_metrics_rmse(cube: iris.cube.Cube): - """Test taking the rmse between two cubes.""" + """Test taking the rmse between two cubes using the metric selector.""" # Data preparation. other_cube = cube.copy() del other_cube.attributes["cset_comparison_base"] @@ -198,6 +198,27 @@ def test_scores_metrics_rmse(cube: iris.cube.Cube): assert rmse_cube.long_name == "RMSE_of_air_temperature" +def test_scores_metrics_unknown(cube: iris.cube.Cube): + """Test calling scores_metrics with an unknown metric.""" + # Data preparation. + other_cube = cube.copy() + del other_cube.attributes["cset_comparison_base"] + cubes = iris.cube.CubeList([cube, other_cube]) + + # Create an empty cubelist. + cubes_empty = iris.cube.CubeList([]) + + # Call the metric selector. + unknown_cube = scoreswrappers.scores_metrics( + cubes, + preserved_coordinates=["time", "grid_latitude", "grid_longitude"], + scores_method="Unknown", + ) + + # As the metric is unknown, the selector should return an empty cubelist. + assert unknown_cube == cubes_empty + + def test_scores_rmse_nonzero(cube: iris.cube.Cube): """Test taking the rmse between two different cubes.""" # Data preparation. From 66ed715dce699316042a29b02a9001feec4956f1 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 2 Jul 2026 09:01:02 +0100 Subject: [PATCH 050/117] Removed rouge test file ~/CSET/src/CSET/tests/test_scoreswrappers.py #2214 --- src/CSET/tests/test_scoreswrappers.py | 72 --------------------------- 1 file changed, 72 deletions(-) delete mode 100644 src/CSET/tests/test_scoreswrappers.py diff --git a/src/CSET/tests/test_scoreswrappers.py b/src/CSET/tests/test_scoreswrappers.py deleted file mode 100644 index 78f2f3f90..000000000 --- a/src/CSET/tests/test_scoreswrappers.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Unit tests for operators/scoreswrappers.py.""" - -from unittest.mock import patch - -import iris -import numpy as np -import pytest - -from CSET.operators import scoreswrappers - - -def create_cube(data, name="test", units="m"): - """Create a simple 1D Iris cube for testing.""" - return iris.cube.Cube( - np.array(data, dtype=float), - long_name=name, - units=units, - dim_coords_and_dims=[ - (iris.coords.DimCoord(np.arange(len(data)), standard_name="latitude"), 0) - ], - ) - - -class TestScoresMetrics: - """Tests for the scores_metrics function.""" - - @pytest.mark.parametrize( - "scores_method,wrapper_name", - [ - ("RMSE", "scores_rmse"), - ("MAE", "scores_mae"), - ("additive_bias", "scores_additive_bias"), - ("correlation_pearsonr", "scores_correlation_pearsonr"), - ], - ) - def test_calls_correct_scores_wrapper(self, scores_method, wrapper_name): - """The scores_metrics dispatcher should call the right wrapper.""" - cube_a = create_cube([1.0, 2.0], name="base") - cube_b = create_cube([1.5, 2.5], name="other") - cubes = iris.cube.CubeList([cube_a, cube_b]) - expected_cube = create_cube([0.0, 0.0], name="expected") - - with patch( - f"CSET.operators.scoreswrappers.{wrapper_name}", - return_value=expected_cube, - ) as mock_wrapper: - result = scoreswrappers.scores_metrics( - cubes, - preserved_coordinates=["time"], - scores_method=scores_method, - ) - - mock_wrapper.assert_called_once_with(cubes, ["time"]) - assert isinstance(result, iris.cube.CubeList) - assert len(result) == 1 - assert result[0] is expected_cube - - @pytest.mark.parametrize("scores_method", [None, "", "invalid_method"]) - def test_returns_empty_cubelist_for_unknown_method(self, scores_method): - """Unknown or missing methods should return an empty CubeList.""" - cube_a = create_cube([1.0]) - cube_b = create_cube([2.0]) - cubes = iris.cube.CubeList([cube_a, cube_b]) - - result = scoreswrappers.scores_metrics( - cubes, - preserved_coordinates=None, - scores_method=scores_method, - ) - - assert isinstance(result, iris.cube.CubeList) - assert len(result) == 0 From a65da5ecc25b8db5b6990911a27291c6a65955ae Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 2 Jul 2026 10:02:52 +0100 Subject: [PATCH 051/117] Removed rogue test file ~/CSET/src/CSET/tests/test_verification.py #2214 --- src/CSET/tests/test_verification.py | 178 ---------------------------- 1 file changed, 178 deletions(-) delete mode 100644 src/CSET/tests/test_verification.py diff --git a/src/CSET/tests/test_verification.py b/src/CSET/tests/test_verification.py deleted file mode 100644 index 219e1fe4f..000000000 --- a/src/CSET/tests/test_verification.py +++ /dev/null @@ -1,178 +0,0 @@ -"""Unit tests for loaders/verification.py.""" - -from unittest.mock import MagicMock, patch - -from CSET.loaders.verification import ( - _get_scores_spatial_methods, - _get_scores_timeseries_methods, - load, -) -from CSET.recipes import Config, RawRecipe - - -class TestGetScoresSpatialMethods: - """Test the _get_scores_spatial_methods helper.""" - - def make_conf(self, **kwargs): - """Test make_conf.""" - conf = MagicMock(spec=Config) - conf.SCORES_SPATIAL_RMSE = kwargs.get("SCORES_SPATIAL_RMSE", False) - conf.SCORES_SPATIAL_AB = kwargs.get("SCORES_SPATIAL_AB", False) - conf.SCORES_SPATIAL_MAE = kwargs.get("SCORES_SPATIAL_MAE", False) - return conf - - def test_returns_empty_list_when_no_spatial_scores_enabled(self): - """Test some code.""" - conf = self.make_conf() - - assert _get_scores_spatial_methods(conf) == [] - - def test_returns_expected_spatial_score_methods(self): - """Test some code.""" - conf = self.make_conf( - SCORES_SPATIAL_RMSE=True, - SCORES_SPATIAL_AB=True, - SCORES_SPATIAL_MAE=True, - ) - - assert _get_scores_spatial_methods(conf) == [ - "RMSE", - "additive_bias", - "MAE", - ] - - -class TestGetScoresTimeseriesMethods: - """Test the _get_scores_timeseries_methods helper.""" - - def make_conf(self, **kwargs): - """Test some code.""" - conf = MagicMock(spec=Config) - conf.SCORES_TIMESERIES_RMSE = kwargs.get("SCORES_TIMESERIES_RMSE", False) - conf.SCORES_TIMESERIES_AB = kwargs.get("SCORES_TIMESERIES_AB", False) - conf.SCORES_TIMESERIES_MAE = kwargs.get("SCORES_TIMESERIES_MAE", False) - conf.SCORES_TIMESERIES_PC = kwargs.get("SCORES_TIMESERIES_PC", False) - return conf - - def test_returns_empty_list_when_no_timeseries_scores_enabled(self): - """Test some code.""" - conf = self.make_conf() - - assert _get_scores_timeseries_methods(conf) == [] - - def test_returns_expected_timeseries_score_methods(self): - """Test some code.""" - conf = self.make_conf( - SCORES_TIMESERIES_RMSE=True, - SCORES_TIMESERIES_AB=True, - SCORES_TIMESERIES_MAE=True, - SCORES_TIMESERIES_PC=True, - ) - - assert _get_scores_timeseries_methods(conf) == [ - "RMSE", - "additive_bias", - "MAE", - "correlation_pearsonr", - ] - - -class TestLoadVerificationRecipes: - """Test the load function in loaders/verification.py.""" - - def make_conf(self, **kwargs): - """Test some code.""" - conf = MagicMock(spec=Config) - conf.SCORES_SPATIAL_RMSE = kwargs.get("SCORES_SPATIAL_RMSE", False) - conf.SCORES_SPATIAL_AB = kwargs.get("SCORES_SPATIAL_AB", False) - conf.SCORES_SPATIAL_MAE = kwargs.get("SCORES_SPATIAL_MAE", False) - conf.SCORES_TIMESERIES_RMSE = kwargs.get("SCORES_TIMESERIES_RMSE", False) - conf.SCORES_TIMESERIES_AB = kwargs.get("SCORES_TIMESERIES_AB", False) - conf.SCORES_TIMESERIES_MAE = kwargs.get("SCORES_TIMESERIES_MAE", False) - conf.SCORES_TIMESERIES_PC = kwargs.get("SCORES_TIMESERIES_PC", False) - conf.SURFACE_FIELDS = kwargs.get("SURFACE_FIELDS", ["temperature"]) - conf.SPATIAL_SURFACE_FIELD_METHOD = kwargs.get( - "SPATIAL_SURFACE_FIELD_METHOD", ["SEQ"] - ) - conf.SELECT_SUBAREA = kwargs.get("SELECT_SUBAREA", False) - conf.SUBAREA_TYPE = kwargs.get("SUBAREA_TYPE", None) - conf.SUBAREA_EXTENT = kwargs.get("SUBAREA_EXTENT", None) - conf.asdict.return_value = kwargs.get("asdict", {}) - return conf - - @patch("CSET.loaders.verification.get_models") - def test_load_yields_no_recipes_when_no_scores_selected(self, mock_get_models): - """Test some code.""" - mock_get_models.return_value = [ - {"name": "base", "id": 1}, - {"name": "other", "id": 2}, - ] - conf = self.make_conf() - - assert list(load(conf)) == [] - mock_get_models.assert_called_once_with(conf.asdict.return_value) - - @patch("CSET.loaders.verification.get_models") - def test_load_yields_spatial_score_recipes(self, mock_get_models): - """Test some code.""" - mock_get_models.return_value = [ - {"name": "base_model", "id": 10}, - {"name": "other_model", "id": 20}, - ] - conf = self.make_conf( - SCORES_SPATIAL_RMSE=True, - SCORES_SPATIAL_MAE=True, - SURFACE_FIELDS=["v1", "v2"], - SPATIAL_SURFACE_FIELD_METHOD=["SEQ", "AVE"], - SELECT_SUBAREA=True, - SUBAREA_TYPE="UK", - SUBAREA_EXTENT=[0, 1, 2, 3], - asdict={"dummy": "value"}, - ) - - recipes = list(load(conf)) - - assert len(recipes) == 8 - for recipe in recipes: - assert isinstance(recipe, RawRecipe) - assert recipe.recipe == "surface_difference_scores.yaml" - assert recipe.variables["BASE_MODEL"] == "base_model" - assert recipe.variables["OTHER_MODEL"] == "other_model" - assert recipe.variables["SUBAREA_TYPE"] == "UK" - assert recipe.variables["SUBAREA_EXTENT"] == [0, 1, 2, 3] - assert recipe.model_ids == [10, 20] - - mock_get_models.assert_called_once_with(conf.asdict.return_value) - - @patch("CSET.loaders.verification.get_models") - def test_load_yields_timeseries_score_recipes(self, mock_get_models): - """Test some code.""" - mock_get_models.return_value = [ - {"name": "reference", "id": 1}, - {"name": "comparison", "id": 2}, - ] - conf = self.make_conf( - SCORES_TIMESERIES_RMSE=True, - SCORES_TIMESERIES_PC=True, - SURFACE_FIELDS=["t2m"], - SELECT_SUBAREA=False, - asdict={"dummy": "value"}, - ) - - recipes = list(load(conf)) - - assert len(recipes) == 2 - assert {recipe.recipe for recipe in recipes} == { - "timeseries_surface_difference_scores.yaml" - } - - for recipe in recipes: - assert isinstance(recipe, RawRecipe) - assert recipe.variables["BASE_MODEL"] == "reference" - assert recipe.variables["OTHER_MODEL"] == "comparison" - assert recipe.variables["SUBAREA_TYPE"] is None - assert recipe.variables["SUBAREA_EXTENT"] is None - assert recipe.model_ids == [1, 2] - - score_methods = {recipe.variables["SCORES_METHOD"] for recipe in recipes} - assert score_methods == {"RMSE", "correlation_pearsonr"} From 49f285ddee122221b1192a10804ee15c29255112 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 08:18:32 +0100 Subject: [PATCH 052/117] Added SATURATION_FRACTION to rose-suite.conf.example #2214 --- src/CSET/cset_workflow/rose-suite.conf.example | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 8cd4ccf17..5bd339a76 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -119,6 +119,7 @@ RAIN_PRESENCE_SPATIAL_DIFFERENCE=False RAIN_PRESENCE_SPATIAL_PLOT=False RUN_METPLUS_GRID_STAT=False RUN_METPLUS_POINT_STAT=False +SATURATION_FRACTION=None SCORES_ALL=False SCORES_SPATIAL_AB=False SCORES_SPATIAL_MAE=False From 8c1a5b7a68327872e5ac3a8a3cf829f260238599 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 08:43:56 +0100 Subject: [PATCH 053/117] Setting SATURATION_FRACTION to False #2214 --- src/CSET/cset_workflow/rose-suite.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 5bd339a76..0f463ff85 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -119,7 +119,7 @@ RAIN_PRESENCE_SPATIAL_DIFFERENCE=False RAIN_PRESENCE_SPATIAL_PLOT=False RUN_METPLUS_GRID_STAT=False RUN_METPLUS_POINT_STAT=False -SATURATION_FRACTION=None +SATURATION_FRACTION=False SCORES_ALL=False SCORES_SPATIAL_AB=False SCORES_SPATIAL_MAE=False From 59df88f07345af7a10a3f2a8707497169429b575 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:18:52 +0100 Subject: [PATCH 054/117] Removed SCORES_METHOD from spatial scores loader #2214 --- src/CSET/loaders/verification.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 3a6a730ac..69a7d1bae 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -68,7 +68,6 @@ def load(conf: Config): "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], "METHOD": method, - "SCORES_METHOD": scores_method, "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA From 953e5a7b8e6fb57270be7267aff50c3f146ee4f1 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:20:11 +0100 Subject: [PATCH 055/117] Hardwired the MAE scores method into this recipe #2214 --- .../recipes/verification/surface_difference_scores_MAE.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml index f78fd7bd8..1af970c75 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" +title: "$METHOD $VARNAME\nScores Mean Absolute Error between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Mean Absolute Error in $METHOD of $VARNAME for all times. The MAE is calculated based on that used in the @@ -34,9 +34,8 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_metrics + - operator: scoreswrappers.scores_mae preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - scores_method: $SCORES_METHOD - operator: collapse.collapse coordinate: ["time"] From d0e195027b0b344f954ed784309997497354592b Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:21:22 +0100 Subject: [PATCH 056/117] Hardwired the RMSE scores method into spatial scores RMSE recipe #2214 --- .../recipes/verification/surface_difference_scores_RMSE.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml index c4459f77c..770861a9b 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" +title: "$METHOD $VARNAME\nScores RMSE between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME for all times. The RMSE is calculated based on that used in the @@ -33,9 +33,8 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_metrics + - operator: scoreswrappers.scores_rmse preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - scores_method: $SCORES_METHOD - operator: collapse.collapse coordinate: ["time"] From 2f055efa6a20f3bfa1b3d596afd3cd2d6160b791 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:21:58 +0100 Subject: [PATCH 057/117] Hardwired the ME scores method into spatial scores ME recipe #2214 --- .../surface_difference_scores_additive_bias.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml index 6b0ae035e..45d90f37a 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" +title: "$METHOD $VARNAME\nScores Mean Error between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Mean Error (also known as the Additive Bias) in $METHOD of $VARNAME for all times. The ME is calculated based on that used in the @@ -32,9 +32,8 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_metrics + - operator: scoreswrappers.scores_additive_bias preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - scores_method: $SCORES_METHOD - operator: collapse.collapse coordinate: ["time"] From 5a4921e3fa5eaa5f42f321709874f3b2c9af2250 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:34:07 +0100 Subject: [PATCH 058/117] Removed SCORES_METHOD from timeseries scores loader #2214 --- src/CSET/loaders/verification.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 69a7d1bae..1966e8a35 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -90,7 +90,6 @@ def load(conf: Config): "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], - "SCORES_METHOD": scores_method, "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA From c464d72d02f4e668d79004720ee6a43b4e85e77c Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:35:01 +0100 Subject: [PATCH 059/117] Hardwired the MAE scores method into timeseries scores MAE recipe #2214 --- .../timeseries_surface_difference_scores_MAE.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml index f2bfb6e46..bc4df9c6c 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores Mean Absolute Error timeseries between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Mean Absolute Error in $VARNAME for all times. The MAE is calculated based on that used in the @@ -34,9 +34,13 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_metrics + - operator: scoreswrappers.scores_mae preserved_coordinates: ["time"] - scores_method: $SCORES_METHOD +# scores_method: $SCORES_METHOD + +# - operator: scoreswrappers.scores_metrics +# preserved_coordinates: ["time"] +# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From 1c28ad3a82fc9b0475cb17ea645f0306f1833b91 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:35:39 +0100 Subject: [PATCH 060/117] Hardwired the RMSE scores method into timeseries scores RMSE recipe #2214 --- .../timeseries_surface_difference_scores_RMSE.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml index 318551f7d..253d6623c 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores RMSE timeseries between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Root Mean Square Error in $VARNAME for all times. The RMSE is calculated based on that used in the @@ -33,9 +33,13 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_metrics + - operator: scoreswrappers.scores_rmse preserved_coordinates: ["time"] - scores_method: $SCORES_METHOD +# scores_method: $SCORES_METHOD + +# - operator: scoreswrappers.scores_metrics +# preserved_coordinates: ["time"] +# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From 34fcf47b3d6dab1d26db47a5e73e3772546a09cf Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:36:17 +0100 Subject: [PATCH 061/117] Hardwired the ME scores method into timeseries scores ME recipe #2214 --- ...series_surface_difference_scores_additive_bias.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml index 1e7d3c9bc..762d5fcb4 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores Mean Error timeseries between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Mean Error (also known as the Additive Bias) in $VARNAME for all times. The ME is calculated based on that used in the @@ -32,9 +32,13 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_metrics + - operator: scoreswrappers.scores_mae preserved_coordinates: ["time"] - scores_method: $SCORES_METHOD +# scores_method: $SCORES_METHOD + +# - operator: scoreswrappers.scores_metrics +# preserved_coordinates: ["time"] +# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From 3b23972190c93f74b0cae4803ce0af86635f9347 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:37:08 +0100 Subject: [PATCH 062/117] Hardwired the Pearsons Correlation scores method into timeseries scores Pearsons Correlation recipe #2214 --- ...surface_difference_scores_correlation_pearsonr.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml index 0253ffc53..17ebb510d 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores Pearson's Correlation timeseries between $OTHER_MODEL and $BASE_MODEL" description: | Extracts and plots the Pearson's Correlation coefficient in $VARNAME for all times. The PC is calculated based on that used in the @@ -31,9 +31,13 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_metrics + - operator: scoreswrappers.scores_correlation_pearsonr preserved_coordinates: ["time"] - scores_method: $SCORES_METHOD +# scores_method: $SCORES_METHOD + +# - operator: scoreswrappers.scores_metrics +# preserved_coordinates: ["time"] +# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From 5b1683eab667e73d58d356d3324f025ef2dea4c2 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:42:30 +0100 Subject: [PATCH 063/117] Tidying the timeseries scores Pearson Correlation recipe #2214 --- ...eries_surface_difference_scores_correlation_pearsonr.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml index 17ebb510d..277e80df6 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml @@ -33,11 +33,6 @@ steps: - operator: scoreswrappers.scores_correlation_pearsonr preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - -# - operator: scoreswrappers.scores_metrics -# preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From 416da3433fc1f6fc1d570fa37c69524c1d22a045 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:43:29 +0100 Subject: [PATCH 064/117] Tidying the timeseries score MAE recipe #2214 --- .../timeseries_surface_difference_scores_MAE.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml index bc4df9c6c..a9501c6fc 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml @@ -36,11 +36,6 @@ steps: - operator: scoreswrappers.scores_mae preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - -# - operator: scoreswrappers.scores_metrics -# preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From 63a12629466fbd06067af27e9ec49ac179d82679 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:44:12 +0100 Subject: [PATCH 065/117] Tidying the timeseries score RMSE recipe #2214 --- .../timeseries_surface_difference_scores_RMSE.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml index 253d6623c..c9f7e04b2 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml @@ -35,11 +35,6 @@ steps: - operator: scoreswrappers.scores_rmse preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - -# - operator: scoreswrappers.scores_metrics -# preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From 906c6ce7660e24d3f95c692bc2b66d89b0516f03 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 3 Jul 2026 16:44:51 +0100 Subject: [PATCH 066/117] Tidying the timeseries score ME recipe #2214 --- ...timeseries_surface_difference_scores_additive_bias.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml index 762d5fcb4..b66d7e0fc 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml @@ -32,13 +32,8 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT - - operator: scoreswrappers.scores_mae + - operator: scoreswrappers.scores_additive_bias preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - -# - operator: scoreswrappers.scores_metrics -# preserved_coordinates: ["time"] -# scores_method: $SCORES_METHOD - operator: plot.plot_line_series series_coordinate: time From f9ccd315f8add5775cb19cd21845b2d026c30441 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 7 Jul 2026 09:43:41 +0100 Subject: [PATCH 067/117] Removed function scores_metrics #2214 --- src/CSET/operators/scoreswrappers.py | 32 ------------- .../surface_difference_scores.yaml | 46 ------------------- .../timeseries_surface_difference_scores.yaml | 44 ------------------ 3 files changed, 122 deletions(-) delete mode 100644 src/CSET/recipes/verification/surface_difference_scores.yaml delete mode 100644 src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 2c69b4fcf..83863ccca 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -244,35 +244,3 @@ def scores_correlation_pearsonr( ) scores_cube.rename(f"Pearson_Correlation_of_{base.name()}") return scores_cube - - -def scores_metrics( - cubes: CubeList, - preserved_coordinates: list[str] | str | None = None, - scores_method: str | None = None, -): - """Select which scores metric to use.""" - # initialise the output cubelist - cubes_output = iris.cube.CubeList([]) - - match scores_method: - case "RMSE": - # Use the RMSE method - return scores_rmse(cubes, preserved_coordinates) - - case "additive_bias": - # Use the Additive Bias (aka mean error) method - return scores_additive_bias(cubes, preserved_coordinates) - - case "MAE": - # Use the MAE method - return scores_mae(cubes, preserved_coordinates) - - case "correlation_pearsonr": - # Use the Pearson's Correlation method - return scores_correlation_pearsonr(cubes, preserved_coordinates) - - case _: - raise ValueError(f"No such scores metric: {scores_method}") - - return cubes_output diff --git a/src/CSET/recipes/verification/surface_difference_scores.yaml b/src/CSET/recipes/verification/surface_difference_scores.yaml deleted file mode 100644 index 2839e083c..000000000 --- a/src/CSET/recipes/verification/surface_difference_scores.yaml +++ /dev/null @@ -1,46 +0,0 @@ -category: Scores -title: "$METHOD $VARNAME\nScores $SCORES_METHOD between $OTHER_MODEL and $BASE_MODEL" -description: | - Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME - for all times. The RMSE is calculated based on that used in the - package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). This recipe preserves the time, latitude and longitude coordinates. - This means a sequence of spatial plots of the RMSE can be produced. - - A larger RMSE implies a greater error than a smaller RMSE. An - RMSE of zero indicates the two field match. The RMSE is calculated - on the grid point and thus a spatial view of the RMSE provides useful - information about where the differences are, or if placement errors - are domininating the score (usually indicated by dipoles). - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: [$BASE_MODEL, $OTHER_MODEL] - constraint: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $VARNAME - pressure_level_constraint: - operator: constraints.generate_level_constraint - coordinate: "pressure" - levels: [] - subarea_type: $SUBAREA_TYPE - subarea_extent: $SUBAREA_EXTENT - - - operator: scoreswrappers.scores_metrics - preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - scores_method: $SCORES_METHOD - - - operator: collapse.collapse - coordinate: ["time"] - method: $METHOD - - - operator: plot.spatial_pcolormesh_plot - - - operator: write.write_cube_to_nc - overwrite: True diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml deleted file mode 100644 index 29b8ab6c0..000000000 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores.yaml +++ /dev/null @@ -1,44 +0,0 @@ -category: Scores -title: "$VARNAME\nScores $SCORES_METHOD timeseries between $OTHER_MODEL and $BASE_MODEL" -description: | - Extracts and plots the Root Mean Square Error in $VARNAME - for all times. The RMSE is calculated based on that used in the - package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). - This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the RMSE is - collapsed over all other coordinates in the cube and calculated for every timestep. - - A larger RMSE implies a greater error than a smaller RMSE. An - RMSE of zero implies the two fields match. The RMSE is calculated - on the grid point and thus a spatial view of the RMSE provides useful - information about where the differences are, or if placement errors - are domininating the score (usually indicated by dipoles). - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: [$BASE_MODEL, $OTHER_MODEL] - constraint: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $VARNAME - pressure_level_constraint: - operator: constraints.generate_level_constraint - coordinate: "pressure" - levels: [] - subarea_type: $SUBAREA_TYPE - subarea_extent: $SUBAREA_EXTENT - - - operator: scoreswrappers.scores_metrics - preserved_coordinates: ["time"] - scores_method: $SCORES_METHOD - - - operator: plot.plot_line_series - series_coordinate: time - - - operator: write.write_cube_to_nc - overwrite: True From d6c765f8fc5bb29653619a45b1ceddfbf27a1d72 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 7 Jul 2026 09:44:22 +0100 Subject: [PATCH 068/117] Removed tests for function scores_metrics #2214 --- tests/operators/test_scoreswrappers.py | 94 -------------------------- 1 file changed, 94 deletions(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 410b84cd5..0c208c692 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -51,34 +51,6 @@ def test_scores_correlation_pearsonr(cube: iris.cube.Cube): ) -def test_scores_metrics_correlation_pearsonr(cube: iris.cube.Cube): - """Test taking the Pearson correlation between two cubes using the metric selector.""" - # Data preparation. - other_cube = cube.copy() - del other_cube.attributes["cset_comparison_base"] - cubes = iris.cube.CubeList([cube, other_cube]) - - # Take difference. - correlation_pearsonr_cube = scoreswrappers.scores_metrics( - cubes, - preserved_coordinates=["time"], - scores_method="correlation_pearsonr", - ) - correlation_pearsonr_cube = scoreswrappers.scores_correlation_pearsonr(cubes) - - # As both cubes use the same data, check the Pearson correlation is one. - assert isinstance(correlation_pearsonr_cube, iris.cube.Cube) - assert np.allclose( - correlation_pearsonr_cube.data - 1.0, - np.zeros_like(correlation_pearsonr_cube.data), - atol=1e-9, - ) - assert correlation_pearsonr_cube.standard_name is None - assert ( - correlation_pearsonr_cube.long_name == "correlation_pearsonr_of_air_temperature" - ) - - def test_scores_additive_bias(cube: iris.cube.Cube): """Test taking the additive bias between two cubes.""" # Data preparation. @@ -98,30 +70,6 @@ def test_scores_additive_bias(cube: iris.cube.Cube): assert additive_bias_cube.long_name == "additive_bias_of_air_temperature" -def test_scores_metrics_additive_bias(cube: iris.cube.Cube): - """Test taking the additive bias between two cubes using the metric selector.""" - # Data preparation. - other_cube = cube.copy() - del other_cube.attributes["cset_comparison_base"] - cubes = iris.cube.CubeList([cube, other_cube]) - - # Take difference. - additive_bias_cube = scoreswrappers.scores_metrics( - cubes, - preserved_coordinates=["time", "grid_latitude", "grid_longitude"], - scores_method="additive_bias", - ) - additive_bias_cube = scoreswrappers.scores_additive_bias(cubes) - - # As both cubes use the same data, check the additive bias is zero. - assert isinstance(additive_bias_cube, iris.cube.Cube) - assert np.allclose( - additive_bias_cube.data, np.zeros_like(additive_bias_cube.data), atol=1e-9 - ) - assert additive_bias_cube.standard_name is None - assert additive_bias_cube.long_name == "additive_bias_of_air_temperature" - - def test_scores_mae(cube: iris.cube.Cube): """Test taking the mae between two cubes.""" # Data preparation. @@ -139,27 +87,6 @@ def test_scores_mae(cube: iris.cube.Cube): assert mae_cube.long_name == "MAE_of_air_temperature" -def test_scores_metrics_mae(cube: iris.cube.Cube): - """Test taking the mae between two cubes using the metric selector.""" - # Data preparation. - other_cube = cube.copy() - del other_cube.attributes["cset_comparison_base"] - cubes = iris.cube.CubeList([cube, other_cube]) - - # Take difference. - mae_cube = scoreswrappers.scores_metrics( - cubes, - preserved_coordinates=["time", "grid_latitude", "grid_longitude"], - scores_method="MAE", - ) - - # As both cubes use the same data, check the mae is zero. - assert isinstance(mae_cube, iris.cube.Cube) - assert np.allclose(mae_cube.data, np.zeros_like(mae_cube.data), atol=1e-9) - assert mae_cube.standard_name is None - assert mae_cube.long_name == "MAE_of_air_temperature" - - def test_scores_rmse(cube: iris.cube.Cube): """Test taking the rmse between two cubes.""" # Data preparation. @@ -198,27 +125,6 @@ def test_scores_metrics_rmse(cube: iris.cube.Cube): assert rmse_cube.long_name == "RMSE_of_air_temperature" -def test_scores_metrics_unknown(cube: iris.cube.Cube): - """Test calling scores_metrics with an unknown metric.""" - # Data preparation. - other_cube = cube.copy() - del other_cube.attributes["cset_comparison_base"] - cubes = iris.cube.CubeList([cube, other_cube]) - - # Create an empty cubelist. - cubes_empty = iris.cube.CubeList([]) - - # Call the metric selector. - unknown_cube = scoreswrappers.scores_metrics( - cubes, - preserved_coordinates=["time", "grid_latitude", "grid_longitude"], - scores_method="Unknown", - ) - - # As the metric is unknown, the selector should return an empty cubelist. - assert unknown_cube == cubes_empty - - def test_scores_rmse_nonzero(cube: iris.cube.Cube): """Test taking the rmse between two different cubes.""" # Data preparation. From e78d55ab6e22e89f123c30fdf362f1c474a1f7c3 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 7 Jul 2026 16:15:50 +0100 Subject: [PATCH 069/117] Trialling new time coords #2214 --- src/CSET/operators/scoreswrappers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 83863ccca..295636b08 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -181,6 +181,8 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = forecasts, predictions or models (2.5.0)". Zenodo. doi: 10.5281/zenodo.18638494 """ base, other = _sort_cubes_for_verification(cubes) + # if preserved_coordinates == ["grid_latitude", "grid_longitude"]: + # time_coord = base.coord[0]("time") # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( @@ -191,6 +193,8 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = ) ) scores_cube.rename(f"RMSE_of_{base.name()}") + # if preserved_coordinates == ["grid_latitude", "grid_longitude"]: + # scores_cube.add_aux_coord(time_coord) return scores_cube From 67f63e4a56a2ecc8d284893d01a55cf9c66bbd8b Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 9 Jul 2026 08:56:07 +0100 Subject: [PATCH 070/117] New recipe to mean over scores RMSE #2214 --- .../surface_difference_scores_MEAN_RMSE.yaml | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml diff --git a/src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml new file mode 100644 index 000000000..0c62e53c8 --- /dev/null +++ b/src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml @@ -0,0 +1,47 @@ +category: Scores +title: "Mean $VARNAME\nScores RMSE between $OTHER_MODEL and $BASE_MODEL" +description: | + Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME + for all times. The RMSE is calculated based on that used in the + package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). + This recipe preserves the time, latitude and longitude coordinates. + This means a sequence of spatial plots of the RMSE can be produced. + + A larger RMSE implies a greater error than a smaller RMSE. An + RMSE of zero indicates the two fields match. The RMSE is calculated + on the grid point and thus a spatial view of the RMSE provides useful + information about where the differences are, or if placement errors + are domininating the score (usually indicated by dipoles). + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: [$BASE_MODEL, $OTHER_MODEL] + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + varname: $VARNAME + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: scoreswrappers.scores_rmse + preserved_coordinates: ["grid_latitude", "grid_longitude"] +# preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + +# - operator: collapse.collapse +# coordinate: ["time"] +# method: $METHOD + + - operator: plot.spatial_pcolormesh_plot + + - operator: write.write_cube_to_nc + overwrite: True From be9aba0ad2a3d04e5d3a52e75bbd2fd3e35f58c8 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 9 Jul 2026 08:57:06 +0100 Subject: [PATCH 071/117] Loader to mean over scores RMSE #2214 --- src/CSET/loaders/verification.py | 51 ++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 1966e8a35..b0b364efc 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -61,21 +61,42 @@ def load(conf: Config): conf.SPATIAL_SURFACE_FIELD_METHOD, scores_spatial_methods, ): - yield RawRecipe( - recipe=f"surface_difference_scores_{scores_method}.yaml", - variables={ - "VARNAME": field, - "BASE_MODEL": base_model["name"], - "OTHER_MODEL": model["name"], - "METHOD": method, - "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, - "SUBAREA_EXTENT": conf.SUBAREA_EXTENT - if conf.SELECT_SUBAREA - else None, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) + if scores_method == "RMSE" and method == "MEAN": + yield RawRecipe( + recipe=f"surface_difference_scores_MEAN_{scores_method}.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "METHOD": method, + "SUBAREA_TYPE": conf.SUBAREA_TYPE + if conf.SELECT_SUBAREA + else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) + else: + yield RawRecipe( + recipe=f"surface_difference_scores_{scores_method}.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "METHOD": method, + "SUBAREA_TYPE": conf.SUBAREA_TYPE + if conf.SELECT_SUBAREA + else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) scores_timeseries_methods = _get_scores_timeseries_methods(conf) if scores_timeseries_methods: From 97a50057ba02199f40f79b11c0ce896b154ab63c Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 9 Jul 2026 09:05:08 +0100 Subject: [PATCH 072/117] Operator to mean over scores RMSE #2214 --- src/CSET/operators/scoreswrappers.py | 88 +++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 3 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 295636b08..7b66cfc10 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -140,6 +140,53 @@ def _sort_cubes_for_verification(cubes: CubeList): return base, other +def _resolve_preserve_dims( + cube: Cube, + data_array: xr.DataArray, + preserved_coordinates: list[str] | str | None, +) -> list[str] | None: + """Resolve preserve coordinates to xarray dimension names. + + The ``scores`` package expects preserve dimensions to match xarray + dimension names. In Iris data, commonly used coordinates such as ``time`` + may be auxiliary coordinates attached to a differently named dimension + (e.g. ``dim0``). This helper maps coordinate names to their underlying + dimension names. + """ + if preserved_coordinates is None: + return None + + coord_names = ( + [preserved_coordinates] + if isinstance(preserved_coordinates, str) + else preserved_coordinates + ) + preserve_dims: list[str] = [] + + for coord_name in coord_names: + # Already an xarray dimension name. + if coord_name in data_array.dims: + if coord_name not in preserve_dims: + preserve_dims.append(coord_name) + continue + + # Otherwise, map coordinate name to dimension index/indices. + try: + dim_indices = cube.coord_dims(coord_name) + except iris.exceptions.CoordinateNotFoundError: + # Keep original name so scores raises a clear error for unknown keys. + if coord_name not in preserve_dims: + preserve_dims.append(coord_name) + continue + + for dim_index in dim_indices: + dim_name = data_array.dims[dim_index] + if dim_name not in preserve_dims: + preserve_dims.append(dim_name) + + return preserve_dims + + def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = None): r"""Calculate the Root Mean Square Error (RMSE) using scores. @@ -181,17 +228,52 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = forecasts, predictions or models (2.5.0)". Zenodo. doi: 10.5281/zenodo.18638494 """ base, other = _sort_cubes_for_verification(cubes) + + # SB + other_xr = xr.DataArray.from_iris(other) + base_xr = xr.DataArray.from_iris(base) + preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) + # if preserved_coordinates == ["grid_latitude", "grid_longitude"]: # time_coord = base.coord[0]("time") # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.rmse( - xr.DataArray.from_iris(other), - xr.DataArray.from_iris(base), - preserve_dims=preserved_coordinates, + other_xr, + base_xr, + preserve_dims=preserve_dims, ) ) + + # If time is aggregated out, attach a scalar time coordinate with bounds + # so plotting can display the aggregated period in the title. + try: + if not scores_cube.coords("time"): + base_time = base.coord("time") + time_vals = ( + base_time.bounds.flatten() + if base_time.has_bounds() + else base_time.points + ) + t_start = float(time_vals[0]) + t_end = float(time_vals[-1]) + t_mid = 0.5 * (t_start + t_end) + + scores_cube.add_aux_coord( + iris.coords.AuxCoord( + t_mid, + standard_name=base_time.standard_name, + long_name=base_time.long_name, + var_name=base_time.var_name, + units=base_time.units, + bounds=np.array([t_start, t_end]), + attributes=base_time.attributes.copy(), + ) + ) + except iris.exceptions.CoordinateNotFoundError: + pass + scores_cube.rename(f"RMSE_of_{base.name()}") # if preserved_coordinates == ["grid_latitude", "grid_longitude"]: # scores_cube.add_aux_coord(time_coord) From 053456c28b4ff0a0b383c919caada478cdf7d1cd Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 9 Jul 2026 16:23:18 +0100 Subject: [PATCH 073/117] Put into scores spatial recipe #2214 --- .../surface_difference_scores_MEAN_RMSE.yaml | 47 ------------------- .../surface_difference_scores_RMSE.yaml | 2 +- 2 files changed, 1 insertion(+), 48 deletions(-) delete mode 100644 src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml diff --git a/src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml deleted file mode 100644 index 0c62e53c8..000000000 --- a/src/CSET/recipes/verification/surface_difference_scores_MEAN_RMSE.yaml +++ /dev/null @@ -1,47 +0,0 @@ -category: Scores -title: "Mean $VARNAME\nScores RMSE between $OTHER_MODEL and $BASE_MODEL" -description: | - Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME - for all times. The RMSE is calculated based on that used in the - package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). - This recipe preserves the time, latitude and longitude coordinates. - This means a sequence of spatial plots of the RMSE can be produced. - - A larger RMSE implies a greater error than a smaller RMSE. An - RMSE of zero indicates the two fields match. The RMSE is calculated - on the grid point and thus a spatial view of the RMSE provides useful - information about where the differences are, or if placement errors - are domininating the score (usually indicated by dipoles). - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: [$BASE_MODEL, $OTHER_MODEL] - constraint: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $VARNAME - pressure_level_constraint: - operator: constraints.generate_level_constraint - coordinate: "pressure" - levels: [] - subarea_type: $SUBAREA_TYPE - subarea_extent: $SUBAREA_EXTENT - - - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["grid_latitude", "grid_longitude"] -# preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - -# - operator: collapse.collapse -# coordinate: ["time"] -# method: $METHOD - - - operator: plot.spatial_pcolormesh_plot - - - operator: write.write_cube_to_nc - overwrite: True diff --git a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml index 770861a9b..72590977b 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml @@ -34,7 +34,7 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + preserved_coordinates: $PRESERVED_COORDS - operator: collapse.collapse coordinate: ["time"] From 3552a04c39d0ff3de427cd6e321df5e64abd2294 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 9 Jul 2026 16:24:11 +0100 Subject: [PATCH 074/117] Tidied scores spatial loader #2214 --- src/CSET/loaders/verification.py | 54 +++++++++++--------------------- 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index b0b364efc..0f7fa99bc 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -61,42 +61,26 @@ def load(conf: Config): conf.SPATIAL_SURFACE_FIELD_METHOD, scores_spatial_methods, ): + preserved_coords = ["time", "grid_latitude", "grid_longitude"] if scores_method == "RMSE" and method == "MEAN": - yield RawRecipe( - recipe=f"surface_difference_scores_MEAN_{scores_method}.yaml", - variables={ - "VARNAME": field, - "BASE_MODEL": base_model["name"], - "OTHER_MODEL": model["name"], - "METHOD": method, - "SUBAREA_TYPE": conf.SUBAREA_TYPE - if conf.SELECT_SUBAREA - else None, - "SUBAREA_EXTENT": conf.SUBAREA_EXTENT - if conf.SELECT_SUBAREA - else None, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) - else: - yield RawRecipe( - recipe=f"surface_difference_scores_{scores_method}.yaml", - variables={ - "VARNAME": field, - "BASE_MODEL": base_model["name"], - "OTHER_MODEL": model["name"], - "METHOD": method, - "SUBAREA_TYPE": conf.SUBAREA_TYPE - if conf.SELECT_SUBAREA - else None, - "SUBAREA_EXTENT": conf.SUBAREA_EXTENT - if conf.SELECT_SUBAREA - else None, - }, - model_ids=[base_model["id"], model["id"]], - aggregation=False, - ) + preserved_coords = ["grid_latitude", "grid_longitude"] + method = "" + yield RawRecipe( + recipe=f"surface_difference_scores_{scores_method}.yaml", + variables={ + "VARNAME": field, + "BASE_MODEL": base_model["name"], + "OTHER_MODEL": model["name"], + "METHOD": method, + "PRESERVED_COORDS": preserved_coords, + "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, + "SUBAREA_EXTENT": conf.SUBAREA_EXTENT + if conf.SELECT_SUBAREA + else None, + }, + model_ids=[base_model["id"], model["id"]], + aggregation=False, + ) scores_timeseries_methods = _get_scores_timeseries_methods(conf) if scores_timeseries_methods: From ad496736c2f8afce6e74c4a1b8273ec126b0404b Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 13 Jul 2026 15:04:38 +0100 Subject: [PATCH 075/117] Updates to verification.py #2214 --- src/CSET/loaders/verification.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 0f7fa99bc..2a8ae681e 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -63,6 +63,8 @@ def load(conf: Config): ): preserved_coords = ["time", "grid_latitude", "grid_longitude"] if scores_method == "RMSE" and method == "MEAN": + # Set the preserved coords and collapse method required + # to produce RMSE spatial plot over an entire case study. preserved_coords = ["grid_latitude", "grid_longitude"] method = "" yield RawRecipe( From 846f16a94c8d23a8804ddd1e4cc59c006d73b88e Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Mon, 13 Jul 2026 15:05:08 +0100 Subject: [PATCH 076/117] Updates scores RMSE recipe #2214 --- .../recipes/verification/surface_difference_scores_RMSE.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml index 72590977b..dc0802949 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml @@ -13,6 +13,9 @@ description: | information about where the differences are, or if placement errors are domininating the score (usually indicated by dipoles). + TODO - total RMSE over case study + + steps: - operator: read.read_cubes file_paths: $INPUT_PATHS From 08245d54d8b68cafd60445d2a7a91f8447bd0b37 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 14 Jul 2026 15:28:21 +0100 Subject: [PATCH 077/117] Updated the scores RMSE spatial plot recipe description #2214 --- .../recipes/verification/surface_difference_scores_RMSE.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml index dc0802949..ccebdc467 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml @@ -13,8 +13,9 @@ description: | information about where the differences are, or if placement errors are domininating the score (usually indicated by dipoles). - TODO - total RMSE over case study - + For RMSE values computed across an entire casse study, the computation + is done within the scores RMSE module, rather than trying to mean across + each timestep. This preserves the nonlinear nature of the RMSE metric. steps: - operator: read.read_cubes From 2ff90505d7c3a8242c64f857f4f89e300e3b60ec Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 14 Jul 2026 16:38:11 +0100 Subject: [PATCH 078/117] Scores operator now implements correct averaging MAE over a case study #2214 --- src/CSET/operators/scoreswrappers.py | 45 ++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 7b66cfc10..1b1af0b2f 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -229,13 +229,11 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = """ base, other = _sort_cubes_for_verification(cubes) - # SB + # Copy the coordinates of the input cubes. other_xr = xr.DataArray.from_iris(other) base_xr = xr.DataArray.from_iris(base) preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) - # if preserved_coordinates == ["grid_latitude", "grid_longitude"]: - # time_coord = base.coord[0]("time") # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( @@ -283,15 +281,50 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = def scores_mae(cubes: CubeList, preserved_coordinates: list[str] | str | None = None): """Calculate the Mean Absolute Error (MAE) using scores.""" base, other = _sort_cubes_for_verification(cubes) + + # Copy the coordinates of the input cubes. + other_xr = xr.DataArray.from_iris(other) + base_xr = xr.DataArray.from_iris(base) + preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) + # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.mae( - xr.DataArray.from_iris(other), - xr.DataArray.from_iris(base), - preserve_dims=preserved_coordinates, + other_xr, + base_xr, + preserve_dims=preserve_dims, ) ) + + # If time is aggregated out, attach a scalar time coordinate with bounds + # so plotting can display the aggregated period in the title. + try: + if not scores_cube.coords("time"): + base_time = base.coord("time") + time_vals = ( + base_time.bounds.flatten() + if base_time.has_bounds() + else base_time.points + ) + t_start = float(time_vals[0]) + t_end = float(time_vals[-1]) + t_mid = 0.5 * (t_start + t_end) + + scores_cube.add_aux_coord( + iris.coords.AuxCoord( + t_mid, + standard_name=base_time.standard_name, + long_name=base_time.long_name, + var_name=base_time.var_name, + units=base_time.units, + bounds=np.array([t_start, t_end]), + attributes=base_time.attributes.copy(), + ) + ) + except iris.exceptions.CoordinateNotFoundError: + pass + scores_cube.rename(f"MAE_of_{base.name()}") return scores_cube From 44ab6623f9157c28c6005ee2dc754ada4361b7f4 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 14 Jul 2026 16:38:40 +0100 Subject: [PATCH 079/117] Scores loader now implements correct averaging MAE over a case study #2214 --- src/CSET/loaders/verification.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 2a8ae681e..88e394b46 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -67,6 +67,11 @@ def load(conf: Config): # to produce RMSE spatial plot over an entire case study. preserved_coords = ["grid_latitude", "grid_longitude"] method = "" + if scores_method == "MAE" and method == "MEAN": + # Set the preserved coords and collapse method required + # to produce MAE spatial plot over an entire case study. + preserved_coords = ["grid_latitude", "grid_longitude"] + method = "" yield RawRecipe( recipe=f"surface_difference_scores_{scores_method}.yaml", variables={ From 0a2d3730d90a99b03d8cb3d7bf80f5a965757358 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Tue, 14 Jul 2026 16:39:18 +0100 Subject: [PATCH 080/117] Scores spatial plot recipe now implements correct averaging MAE over a case study #2214 --- .../verification/surface_difference_scores_MAE.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml index 1af970c75..9954ba889 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml @@ -14,6 +14,10 @@ description: | are domininating the score (usually indicated by dipoles). MAE is a fair measure of error when forecasting the median and is not sensitive to outliers. + For MAE values computed across an entire casse study, the computation + is done within the scores MAE module, rather than trying to mean across + each timestep. This preserves the nonlinear nature of the MAE metric. + steps: - operator: read.read_cubes file_paths: $INPUT_PATHS @@ -35,7 +39,8 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_mae - preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + preserved_coordinates: $PRESERVED_COORDS +# preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - operator: collapse.collapse coordinate: ["time"] From 838bba3383554f183344ef009c04363eef841cd9 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Wed, 15 Jul 2026 10:20:36 +0100 Subject: [PATCH 081/117] adding conversion from iris to xarray dimension coordinates to all scores error metrics for spatial surface plots. --- src/CSET/operators/scoreswrappers.py | 81 +++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 6 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 1b1af0b2f..5786b4576 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -334,15 +334,49 @@ def scores_additive_bias( ): """Calculate the Additive Bias (Mean Error) using scores.""" base, other = _sort_cubes_for_verification(cubes) + + # Copy the coordinates of the input cubes. + other_xr = xr.DataArray.from_iris(other) + base_xr = xr.DataArray.from_iris(base) + preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) + # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.additive_bias( - xr.DataArray.from_iris(other), - xr.DataArray.from_iris(base), - preserve_dims=preserved_coordinates, + other_xr, + base_xr, + preserve_dims=preserve_dims, ) ) + + # If time is aggregated out, attach a scalar time coordinate with bounds + # so plotting can display the aggregated period in the title. + try: + if not scores_cube.coords("time"): + base_time = base.coord("time") + time_vals = ( + base_time.bounds.flatten() + if base_time.has_bounds() + else base_time.points + ) + t_start = float(time_vals[0]) + t_end = float(time_vals[-1]) + t_mid = 0.5 * (t_start + t_end) + + scores_cube.add_aux_coord( + iris.coords.AuxCoord( + t_mid, + standard_name=base_time.standard_name, + long_name=base_time.long_name, + var_name=base_time.var_name, + units=base_time.units, + bounds=np.array([t_start, t_end]), + attributes=base_time.attributes.copy(), + ) + ) + except iris.exceptions.CoordinateNotFoundError: + pass scores_cube.rename(f"Additive_Bias_of_{base.name()}") return scores_cube @@ -352,14 +386,49 @@ def scores_correlation_pearsonr( ): """Calculate the Pearson's Correlation using scores.""" base, other = _sort_cubes_for_verification(cubes) + + # Copy the coordinates of the input cubes. + other_xr = xr.DataArray.from_iris(other) + base_xr = xr.DataArray.from_iris(base) + preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) + # Scores operators on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.correlation.pearsonr( - xr.DataArray.from_iris(other), - xr.DataArray.from_iris(base), - preserve_dims=preserved_coordinates, + other_xr, + base_xr, + preserve_dims=preserve_dims, ) ) + + # If time is aggregated out, attach a scalar time coordinate with bounds + # so plotting can display the aggregated period in the title. + try: + if not scores_cube.coords("time"): + base_time = base.coord("time") + time_vals = ( + base_time.bounds.flatten() + if base_time.has_bounds() + else base_time.points + ) + t_start = float(time_vals[0]) + t_end = float(time_vals[-1]) + t_mid = 0.5 * (t_start + t_end) + + scores_cube.add_aux_coord( + iris.coords.AuxCoord( + t_mid, + standard_name=base_time.standard_name, + long_name=base_time.long_name, + var_name=base_time.var_name, + units=base_time.units, + bounds=np.array([t_start, t_end]), + attributes=base_time.attributes.copy(), + ) + ) + except iris.exceptions.CoordinateNotFoundError: + pass + scores_cube.rename(f"Pearson_Correlation_of_{base.name()}") return scores_cube From eb11c394c9fa874145927dfabf1e9042719625d6 Mon Sep 17 00:00:00 2001 From: Sylviabohnenstengel Date: Wed, 15 Jul 2026 10:28:32 +0100 Subject: [PATCH 082/117] scoreswrapper typo correction. --- src/CSET/operators/scoreswrappers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 5786b4576..fa5544971 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -234,7 +234,7 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = base_xr = xr.DataArray.from_iris(base) preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) - # Scores operators on xarray data arrays, so we transform the iris cube into an array, + # Scores operates on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.rmse( @@ -287,7 +287,7 @@ def scores_mae(cubes: CubeList, preserved_coordinates: list[str] | str | None = base_xr = xr.DataArray.from_iris(base) preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) - # Scores operators on xarray data arrays, so we transform the iris cube into an array, + # Scores operates on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.mae( @@ -340,7 +340,7 @@ def scores_additive_bias( base_xr = xr.DataArray.from_iris(base) preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) - # Scores operators on xarray data arrays, so we transform the iris cube into an array, + # Scores operates on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.additive_bias( @@ -392,7 +392,7 @@ def scores_correlation_pearsonr( base_xr = xr.DataArray.from_iris(base) preserve_dims = _resolve_preserve_dims(other, other_xr, preserved_coordinates) - # Scores operators on xarray data arrays, so we transform the iris cube into an array, + # Scores operates on xarray data arrays, so we transform the iris cube into an array, # apply scores, and then transform it back. scores_cube = xr.DataArray.to_iris( scores.continuous.correlation.pearsonr( From 06da2fc654321f6c0766718a1ee907713335db47 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 13:14:41 +0100 Subject: [PATCH 083/117] Support for spatial case averaging in scores loaders #2214 --- src/CSET/loaders/verification.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 88e394b46..bbc325a21 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -62,16 +62,24 @@ def load(conf: Config): scores_spatial_methods, ): preserved_coords = ["time", "grid_latitude", "grid_longitude"] - if scores_method == "RMSE" and method == "MEAN": + method_null = "" + scores_method_total = "MEAN" + scores_coords_total = ["grid_latitude", "grid_longitude"] + if scores_method == "RMSE" and method == scores_method_total: # Set the preserved coords and collapse method required # to produce RMSE spatial plot over an entire case study. - preserved_coords = ["grid_latitude", "grid_longitude"] - method = "" - if scores_method == "MAE" and method == "MEAN": + preserved_coords = scores_coords_total + method = method_null + if scores_method == "MAE" and method == scores_method_total: # Set the preserved coords and collapse method required # to produce MAE spatial plot over an entire case study. - preserved_coords = ["grid_latitude", "grid_longitude"] - method = "" + preserved_coords = scores_coords_total + method = method_null + if scores_method == "additive_bias" and method == scores_method_total: + # Set the preserved coords and collapse method required + # to produce ME additive bias spatial plot over an entire case study. + preserved_coords = scores_coords_total + method = method_null yield RawRecipe( recipe=f"surface_difference_scores_{scores_method}.yaml", variables={ From 6a30b3cdda5f9de8e98a52e4296b819a29cb8d20 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 13:15:10 +0100 Subject: [PATCH 084/117] Support for spatial case averaging in scores MAE recipe #2214 --- src/CSET/recipes/verification/surface_difference_scores_MAE.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml index 9954ba889..ebc31c7bd 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml @@ -40,7 +40,6 @@ steps: - operator: scoreswrappers.scores_mae preserved_coordinates: $PRESERVED_COORDS -# preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - operator: collapse.collapse coordinate: ["time"] From 08a614d6f674d5f4c7648f4a26c3cfc12b99ae86 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 13:15:37 +0100 Subject: [PATCH 085/117] Support for spatial case averaging in scores ME recipe #2214 --- .../verification/surface_difference_scores_additive_bias.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml index 45d90f37a..b6dd31ca8 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml @@ -33,7 +33,7 @@ steps: subarea_extent: $SUBAREA_EXTENT - operator: scoreswrappers.scores_additive_bias - preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] + preserved_coordinates: $PRESERVED_COORDS - operator: collapse.collapse coordinate: ["time"] From f678497e280a3b5bf92ce96d200de940bce2e083 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 13:58:17 +0100 Subject: [PATCH 086/117] Support in scores loaders for SUBAREA_NAME #2214 --- src/CSET/loaders/verification.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index bbc325a21..c0f11a09e 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -88,6 +88,7 @@ def load(conf: Config): "OTHER_MODEL": model["name"], "METHOD": method, "PRESERVED_COORDS": preserved_coords, + "SUBAREA_NAME": conf.SUBAREA_NAME if conf.SELECT_SUBAREA else "", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA @@ -110,6 +111,7 @@ def load(conf: Config): "VARNAME": field, "BASE_MODEL": base_model["name"], "OTHER_MODEL": model["name"], + "SUBAREA_NAME": conf.SUBAREA_NAME if conf.SELECT_SUBAREA else "", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT if conf.SELECT_SUBAREA From eb1240ada5a7ff04d15eb01bd4bf33efaf924d06 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 13:59:06 +0100 Subject: [PATCH 087/117] Support in scores spatial MAE recipe for SUBAREA_NAME #2214 --- .../recipes/verification/surface_difference_scores_MAE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml index ebc31c7bd..1c7a91050 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$METHOD $VARNAME\nScores Mean Absolute Error between $OTHER_MODEL and $BASE_MODEL" +title: "$METHOD $VARNAME\nScores Mean Absolute Error between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Mean Absolute Error in $METHOD of $VARNAME for all times. The MAE is calculated based on that used in the From 0c0112037951fe36c900939045b9a9fdee312c17 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 13:59:35 +0100 Subject: [PATCH 088/117] Support in scores spatial RMSE recipe for SUBAREA_NAME #2214 --- .../recipes/verification/surface_difference_scores_RMSE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml index ccebdc467..81032ec2c 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$METHOD $VARNAME\nScores RMSE between $OTHER_MODEL and $BASE_MODEL" +title: "$METHOD $VARNAME\nScores RMSE between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME for all times. The RMSE is calculated based on that used in the From be12c876b81fcb67bb612e6546e4d9c3151a3ab8 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 13:59:58 +0100 Subject: [PATCH 089/117] Support in scores spatial ME recipe for SUBAREA_NAME #2214 --- .../verification/surface_difference_scores_additive_bias.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml index b6dd31ca8..6e16364a7 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$METHOD $VARNAME\nScores Mean Error between $OTHER_MODEL and $BASE_MODEL" +title: "$METHOD $VARNAME\nScores Mean Error between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Mean Error (also known as the Additive Bias) in $METHOD of $VARNAME for all times. The ME is calculated based on that used in the From 4050da6e0571372a39e6a75c765c9373ec43d29b Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 14:00:31 +0100 Subject: [PATCH 090/117] Support in scores timeseries MAE recipe for SUBAREA_NAME #2214 --- .../verification/timeseries_surface_difference_scores_MAE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml index a9501c6fc..95afdf521 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores Mean Absolute Error timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores Mean Absolute Error timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Mean Absolute Error in $VARNAME for all times. The MAE is calculated based on that used in the From d039477cdd3e48a0536c53ad540f139832b07a27 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 14:01:00 +0100 Subject: [PATCH 091/117] Support in scores timeseries RMSE recipe for SUBAREA_NAME #2214 --- .../verification/timeseries_surface_difference_scores_RMSE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml index c9f7e04b2..abb31faa4 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores RMSE timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores RMSE timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Root Mean Square Error in $VARNAME for all times. The RMSE is calculated based on that used in the From 160795bf24d60919ff974c9e96ff12922c076686 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 14:01:28 +0100 Subject: [PATCH 092/117] Support in scores timeseries ME recipe for SUBAREA_NAME #2214 --- .../timeseries_surface_difference_scores_additive_bias.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml index b66d7e0fc..8399bc58e 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores Mean Error timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores Mean Error timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Mean Error (also known as the Additive Bias) in $VARNAME for all times. The ME is calculated based on that used in the From fe1881e8f430b84810ca165d987c22912f75ff0f Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 14:02:09 +0100 Subject: [PATCH 093/117] Support in scores timeseries persons correlation recipe for SUBAREA_NAME #2214 --- ...meseries_surface_difference_scores_correlation_pearsonr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml index 277e80df6..477999c89 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml @@ -1,5 +1,5 @@ category: Scores -title: "$VARNAME\nScores Pearson's Correlation timeseries between $OTHER_MODEL and $BASE_MODEL" +title: "$VARNAME\nScores Pearson's Correlation timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Pearson's Correlation coefficient in $VARNAME for all times. The PC is calculated based on that used in the From 286c05f53b2f91c8be1c5e1b754b1efb4354db4d Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 14:33:36 +0100 Subject: [PATCH 094/117] Added SPATIAL_SCORES_FIELD_METHOD into spatial scores GUI #2214 --- .../cset_workflow/meta/verification/rose-meta.conf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index b2e5c6de1..6ff018f5a 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -76,6 +76,16 @@ ns=Verification/ScoresSpatial sort-key=scoresspatial title=Scores spatial +[template variables=SPATIAL_SCORES_FIELD_METHOD] +ns=Verification/ScoresSpatial +description=Select analysis method(s) for output mapped plots. Add all options required. + Leave blank or set to "SEQ" (sequence) for plots each diagnostic output time. + For time-collapsed outputs over each analysis period, set to "CASE", "MAX", "MIN" etc. + Selecting "CASE" will compute the spatial scores metrics over the entire case study. +help=Quoted analysis methods. Settings should be based on available iris.analysis methods for collapsing cube dimensions. +type=python_list +compulsory=true +sort-key=sc-sp-0 [template variables=SCORES_SPATIAL_RMSE] ns=Verification/ScoresSpatial From 251d5a8b7b8e435b5e801e9caef91c0fe561ade6 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 14:34:06 +0100 Subject: [PATCH 095/117] Added SPATIAL_SCORES_FIELD_METHOD into spatial scores loaders #2214 --- src/CSET/loaders/verification.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index c0f11a09e..f0c411263 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -58,27 +58,27 @@ def load(conf: Config): for model, field, method, scores_method in itertools.product( models[1:], conf.SURFACE_FIELDS, - conf.SPATIAL_SURFACE_FIELD_METHOD, + conf.SPATIAL_SCORES_FIELD_METHOD, scores_spatial_methods, ): preserved_coords = ["time", "grid_latitude", "grid_longitude"] method_null = "" - scores_method_total = "MEAN" - scores_coords_total = ["grid_latitude", "grid_longitude"] - if scores_method == "RMSE" and method == scores_method_total: + scores_method_case = "CASE" + scores_coords_case = ["grid_latitude", "grid_longitude"] + if scores_method == "RMSE" and method == scores_method_case: # Set the preserved coords and collapse method required # to produce RMSE spatial plot over an entire case study. - preserved_coords = scores_coords_total + preserved_coords = scores_coords_case method = method_null - if scores_method == "MAE" and method == scores_method_total: + if scores_method == "MAE" and method == scores_method_case: # Set the preserved coords and collapse method required # to produce MAE spatial plot over an entire case study. - preserved_coords = scores_coords_total + preserved_coords = scores_coords_case method = method_null - if scores_method == "additive_bias" and method == scores_method_total: + if scores_method == "additive_bias" and method == scores_method_case: # Set the preserved coords and collapse method required # to produce ME additive bias spatial plot over an entire case study. - preserved_coords = scores_coords_total + preserved_coords = scores_coords_case method = method_null yield RawRecipe( recipe=f"surface_difference_scores_{scores_method}.yaml", From a29b37f5e46aeb5d512d4f6a450c5b45c7b6b1e0 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Wed, 15 Jul 2026 14:47:19 +0100 Subject: [PATCH 096/117] Update src/CSET/loaders/verification.py Co-authored-by: Sylvia Bohnenstengel <62748926+Sylviabohnenstengel@users.noreply.github.com> --- src/CSET/loaders/verification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index f0c411263..02f324fce 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -100,7 +100,7 @@ def load(conf: Config): scores_timeseries_methods = _get_scores_timeseries_methods(conf) if scores_timeseries_methods: - # Produce timeseries plots of scores metrics averaged over the domain. + # Produce timeseries plots of scores metrics averaged over the domain for each case study. base_model = models[0] for model, field, scores_method in itertools.product( models[1:], conf.SURFACE_FIELDS, scores_timeseries_methods From 46d41a5a4c005422ef10704092fd054e0329188d Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Wed, 15 Jul 2026 14:47:39 +0100 Subject: [PATCH 097/117] Update src/CSET/operators/scoreswrappers.py Co-authored-by: Sylvia Bohnenstengel <62748926+Sylviabohnenstengel@users.noreply.github.com> --- src/CSET/operators/scoreswrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index fa5544971..12e73df6a 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -151,7 +151,7 @@ def _resolve_preserve_dims( dimension names. In Iris data, commonly used coordinates such as ``time`` may be auxiliary coordinates attached to a differently named dimension (e.g. ``dim0``). This helper maps coordinate names to their underlying - dimension names. + dimension names and helps to convert from iris to xarray coordinate dimension names. """ if preserved_coordinates is None: return None From 6f5698ea92b8524c4e4651df5de97c69c1e227a9 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Wed, 15 Jul 2026 14:47:58 +0100 Subject: [PATCH 098/117] Update src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml Co-authored-by: Sylvia Bohnenstengel <62748926+Sylviabohnenstengel@users.noreply.github.com> --- .../verification/surface_difference_scores_additive_bias.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml index 6e16364a7..7fd3ca727 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml @@ -2,7 +2,7 @@ category: Scores title: "$METHOD $VARNAME\nScores Mean Error between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Mean Error (also known as the Additive Bias) in $METHOD of $VARNAME - for all times. The ME is calculated based on that used in the + for all times for each case study separately. The ME is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mean_error). This recipe preserves the time, latitude and longitude coordinates. This means a sequence of spatial plots of the ME can be produced. From 2e6afc703d173b789dc4e72599f80a8291822a4c Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Wed, 15 Jul 2026 14:48:14 +0100 Subject: [PATCH 099/117] Update src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml Co-authored-by: Sylvia Bohnenstengel <62748926+Sylviabohnenstengel@users.noreply.github.com> --- .../verification/surface_difference_scores_additive_bias.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml index 7fd3ca727..1d0575065 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml @@ -6,6 +6,7 @@ description: | package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mean_error). This recipe preserves the time, latitude and longitude coordinates. This means a sequence of spatial plots of the ME can be produced. + Currently only working for deterministic models. A larger ME implies a greater error than a smaller ME. Although a near zero ME could be indicative of a good match between two fields, this could also arise if there are large compensating From 308068bf027fbaed2be78cdfc050cf3f8f987581 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 15:12:18 +0100 Subject: [PATCH 100/117] Updated rose-suite.conf.example with SPATIAL_SCORES_FIELD_METHOD #2214 --- src/CSET/cset_workflow/rose-suite.conf.example | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 31bb00800..d65139909 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -167,6 +167,7 @@ SPATIAL_MULTI_VARIABLE=False SPATIAL_PLEVEL_FIELD=False SPATIAL_PLEVEL_FIELD_AGGREGATION=False,False,False,False SPATIAL_PLEVEL_FIELD_METHOD=[""] +SPATIAL_SCORES_FIELD_METHOD=[""] SPATIAL_STRUCTURAL_SIMILARITY_MLEVEL=False SPATIAL_STRUCTURAL_SIMILARITY_PLEVEL_FIELD=False SPATIAL_STRUCTURAL_SIMILARITY_SURFACE_FIELD=False From 08ed06c210dabd8bbbed4517b9bd3ef7a7dd8722 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Wed, 15 Jul 2026 15:24:52 +0100 Subject: [PATCH 101/117] Tidied the scores metrics descriptions in rose-meta.conf #2214 --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 6ff018f5a..f5cffb28a 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -94,7 +94,7 @@ description=SCORES COUNTINOUS METRIC. Plots a 2D spatial plot of the RMSE between two models. The computation is implemented by the scores package. For the time coordinate, the method(s) set in - SPATIAL_SURFACE_FIELD_METHOD will be used. + SPATIAL_SCORES_FIELD_METHOD will be used. help=This computes the metric using the scores package. Details of this can be found at https://scores.readthedocs.io/en/stable/included.html#continuous type=python_boolean @@ -108,7 +108,7 @@ description=SCORES COUNTINOUS METRIC. Plots a 2D spatial plot of the MAE between two models. The computation is implemented by the scores package. For the time coordinate, the method(s) set in - SPATIAL_SURFACE_FIELD_METHOD will be used. + SPATIAL_SCORES_FIELD_METHOD will be used. help=This computes the metric using the scores package. Details of this can be found at https://scores.readthedocs.io/en/stable/included.html#continuous type=python_boolean @@ -123,7 +123,7 @@ description=SCORES COUNTINOUS METRIC. The Additive Bias is also known as the Mean Error. The computation is implemented by the scores package. For the time coordinate, the method(s) set in - SPATIAL_SURFACE_FIELD_METHOD will be used. + SPATIAL_SCORES_FIELD_METHOD will be used. help=This computes the metric using the scores package. Details of this can be found at https://scores.readthedocs.io/en/stable/included.html#continuous type=python_boolean @@ -145,8 +145,6 @@ title=Root Mean Square Error description=SCORES COUNTINOUS METRIC. Plots a timeseries plot of the RMSE between two models. The computation is implemented by the scores package. - For the time coordinate, the method(s) set in - SPATIAL_SURFACE_FIELD_METHOD will be used. help=This computes the metric using the scores package. Details of this can be found at https://scores.readthedocs.io/en/stable/included.html#continuous type=python_boolean From 90d7f483d349730624bd06a1fad52b099a2ece7d Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Thu, 16 Jul 2026 11:37:12 +0100 Subject: [PATCH 102/117] Removed spurious return from scores_correlation_pearsonr #2214 --- src/CSET/operators/scoreswrappers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 8f5c036e3..876b41455 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -438,8 +438,6 @@ def scores_correlation_pearsonr( scores_cube.rename(f"Pearson_Correlation_of_{base.name()}") return scores_cube - RMSE.rename(f"RMSE_of_{base.name()}") - return RMSE def scores_crps_for_ensemble( From 94e6dd6d69e4e6e33c87cbfa42e339a9ade3a61a Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:25:42 +0100 Subject: [PATCH 103/117] Improved description of averaging in scores MAE 2d plot #2214 --- .../surface_difference_scores_MAE.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml index 1c7a91050..cc77db011 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_MAE.yaml @@ -1,12 +1,20 @@ category: Scores title: "$METHOD $VARNAME\nScores Mean Absolute Error between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | - Extracts and plots the Mean Absolute Error in $METHOD of $VARNAME - for all times. The MAE is calculated based on that used in the + Extracts and plots the Mean Absolute Error in $METHOD of $VARNAME. + The MAE is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mae). This recipe preserves the time, latitude and longitude coordinates. This means a sequence of spatial plots of the MAE can be produced. + This recipe can preserve the time, latitude and longitude coordinates, in which case + a sequence of spatial plots of the MAE are produced according to the averaging + method i.e. SEQ, MAX and MIN. + + Alternatively, the MAE can be computed over all time points in the case study, i.e. + preserving just the latitude and longitude coordinates by selecting the method CASE. + This will produce a single output plot of the total MAE. + A larger MAE implies a greater error than a smaller MAE. An RMSE of zero indicates the two fields most likely match. The MAE is calculated on the grid point and thus a spatial view of the MAE provides useful @@ -38,9 +46,13 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT + # The preserved coordinates specifies whether or not the + # MAE is to be computed over the entire case study. - operator: scoreswrappers.scores_mae preserved_coordinates: $PRESERVED_COORDS + # This collapse is relevant only if the MAE has not been + # computed across the entire case study. - operator: collapse.collapse coordinate: ["time"] method: $METHOD From 22fcdd2a606b9691ff25af1d6572c0e5b80b80c1 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:26:23 +0100 Subject: [PATCH 104/117] Improved description of averaging in scores RMSE 2d plot #2214 --- .../surface_difference_scores_RMSE.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml index 81032ec2c..9b4642cf4 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_RMSE.yaml @@ -1,11 +1,17 @@ category: Scores title: "$METHOD $VARNAME\nScores RMSE between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | - Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME - for all times. The RMSE is calculated based on that used in the + Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME. + The RMSE is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). - This recipe preserves the time, latitude and longitude coordinates. - This means a sequence of spatial plots of the RMSE can be produced. + + This recipe can preserve the time, latitude and longitude coordinates, in which case + a sequence of spatial plots of the RMSE are produced according to the averaging + method i.e. SEQ, MAX and MIN. + + Alternatively, the RMSE can be computed over all time points in the case study, i.e. + preserving just the latitude and longitude coordinates, by selecting the method CASE. + This will produce a single output plot of the total RMSE. A larger RMSE implies a greater error than a smaller RMSE. An RMSE of zero indicates the two fields match. The RMSE is calculated @@ -37,9 +43,13 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT + # The preserved coordinates specifies whether or not the + # RMSE is to be computed over the entire case study. - operator: scoreswrappers.scores_rmse preserved_coordinates: $PRESERVED_COORDS + # This collapse is relevant only if the RMSE has not been + # computed across the entire case study. - operator: collapse.collapse coordinate: ["time"] method: $METHOD From 7d54bcbbba230f0ff9497ea30248cf2dfb8b40a9 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:28:03 +0100 Subject: [PATCH 105/117] Improved description of averaging in scores ME 2d plot #2214 --- .../surface_difference_scores_additive_bias.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml index 1d0575065..9cc6b22ca 100644 --- a/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/surface_difference_scores_additive_bias.yaml @@ -1,13 +1,21 @@ category: Scores title: "$METHOD $VARNAME\nScores Mean Error between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | - Extracts and plots the Mean Error (also known as the Additive Bias) in $METHOD of $VARNAME - for all times for each case study separately. The ME is calculated based on that used in the + Extracts and plots the Mean Error (also known as the Additive Bias) in $METHOD of $VARNAME. + The ME is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mean_error). This recipe preserves the time, latitude and longitude coordinates. This means a sequence of spatial plots of the ME can be produced. Currently only working for deterministic models. + This recipe can preserve the time, latitude and longitude coordinates, in which case + a sequence of spatial plots of the ME are produced according to the averaging + method i.e. SEQ, MAX and MIN. + + Alternatively, the ME can be computed over all time points in the case study, i.e. + preserving just the latitude and longitude coordinates by selecting the method CASE. + This will produce a single output plot of the total ME. + A larger ME implies a greater error than a smaller ME. Although a near zero ME could be indicative of a good match between two fields, this could also arise if there are large compensating positive and negative errors. The ME is calculated on the grid point and thus a spatial view of @@ -33,9 +41,13 @@ steps: subarea_type: $SUBAREA_TYPE subarea_extent: $SUBAREA_EXTENT + # The preserved coordinates specifies whether or not the + # ME is to be computed over the entire case study. - operator: scoreswrappers.scores_additive_bias preserved_coordinates: $PRESERVED_COORDS + # This collapse is relevant only if the ME has not been + # computed across the entire case study. - operator: collapse.collapse coordinate: ["time"] method: $METHOD From 040058603ee8c2405c8e4ac9c1a0e02ada2b91fe Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:28:45 +0100 Subject: [PATCH 106/117] Improved description of averaging in scores MAE timeseries plot #2214 --- .../verification/timeseries_surface_difference_scores_MAE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml index 95afdf521..f58ac8d6d 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml @@ -2,7 +2,7 @@ category: Scores title: "$VARNAME\nScores Mean Absolute Error timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Mean Absolute Error in $VARNAME - for all times. The MAE is calculated based on that used in the + computed over the domain for each timestep. The MAE is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mae). This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the MAE is collapsed over all other coordinates in the cube and calculated for every timestep. From 621bd7d41c776c2d5d83d2a87bc9e7422a51c7a2 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:29:40 +0100 Subject: [PATCH 107/117] Improved description of averaging in scores RMSE timeseries plot #2214 --- .../verification/timeseries_surface_difference_scores_RMSE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml index abb31faa4..7915c332d 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_RMSE.yaml @@ -2,7 +2,7 @@ category: Scores title: "$VARNAME\nScores RMSE timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Root Mean Square Error in $VARNAME - for all times. The RMSE is calculated based on that used in the + computed over the domain for each timestep. The RMSE is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the RMSE is collapsed over all other coordinates in the cube and calculated for every timestep. From 137c325dbcec4a872efea6b748d0e6e720e46657 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:30:14 +0100 Subject: [PATCH 108/117] Improved description of averaging in scores ME timeseries plot #2214 --- .../timeseries_surface_difference_scores_additive_bias.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml index 8399bc58e..df9ee7c15 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_additive_bias.yaml @@ -2,7 +2,7 @@ category: Scores title: "$VARNAME\nScores Mean Error timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Mean Error (also known as the Additive Bias) in $VARNAME - for all times. The ME is calculated based on that used in the + computed over the domain for each timestep. The ME is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.mean_error). This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the ME is collapsed over all other coordinates in the cube and calculated for every timestep. From d75820f322140a4bde9a886ed9be06c26681d5c3 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:31:01 +0100 Subject: [PATCH 109/117] Improved description of averaging in scores Pearson Correlation timeseries plot #2214 --- ...meseries_surface_difference_scores_correlation_pearsonr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml index 477999c89..6c4642853 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml @@ -2,7 +2,7 @@ category: Scores title: "$VARNAME\nScores Pearson's Correlation timeseries between $OTHER_MODEL and $BASE_MODEL $SUBAREA_NAME" description: | Extracts and plots the Pearson's Correlation coefficient in $VARNAME - for all times. The PC is calculated based on that used in the + computed over the domain for each timestep. The PC is calculated based on that used in the package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.correlation.pearsonr). This recipe allows the preservation of the time coordinate to produce a timeseries. Therefore, the PC is collapsed over all other coordinates in the cube and calculated for every timestep. From 41eb73d4f07f010cbed8567848f8023c4a5c083e Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:37:34 +0100 Subject: [PATCH 110/117] Minor tidy of scores recipe #2214 --- ...imeseries_surface_difference_scores_correlation_pearsonr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml index 6c4642853..179a735f9 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_correlation_pearsonr.yaml @@ -34,6 +34,7 @@ steps: - operator: scoreswrappers.scores_correlation_pearsonr preserved_coordinates: ["time"] + # Plot a timeseries of the Pearson Correlation - operator: plot.plot_line_series series_coordinate: time From c92675aee25b4d2573267e66b1248f8ce90d04c9 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:44:33 +0100 Subject: [PATCH 111/117] Removed redundant function scores_metrics from testing #1759 --- tests/operators/test_scoreswrappers.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index c9b78aba6..dc033e291 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -112,27 +112,6 @@ def test_scores_rmse(cube: iris.cube.Cube): assert rmse_cube.long_name == "RMSE_of_air_temperature" -def test_scores_metrics_rmse(cube: iris.cube.Cube): - """Test taking the rmse between two cubes using the metric selector.""" - # Data preparation. - other_cube = cube.copy() - del other_cube.attributes["cset_comparison_base"] - cubes = iris.cube.CubeList([cube, other_cube]) - - # Take difference. - rmse_cube = scoreswrappers.scores_metrics( - cubes, - preserved_coordinates=["time", "grid_latitude", "grid_longitude"], - scores_method="RMSE", - ) - - # As both cubes use the same data, check the rmse is zero. - assert isinstance(rmse_cube, iris.cube.Cube) - assert np.allclose(rmse_cube.data, np.zeros_like(rmse_cube.data), atol=1e-9) - assert rmse_cube.standard_name is None - assert rmse_cube.long_name == "RMSE_of_air_temperature" - - def test_scores_rmse_nonzero(cube: iris.cube.Cube): """Test taking the rmse between two different cubes.""" # Data preparation. From 52379444a4279b6c548aa71f7bbe947691313f40 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:50:48 +0100 Subject: [PATCH 112/117] Typo in scores metrics testing of cube names #1759 --- tests/operators/test_scoreswrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index dc033e291..ceb42e5e0 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -55,7 +55,7 @@ def test_scores_correlation_pearsonr(cube: iris.cube.Cube): ) assert correlation_pearsonr_cube.standard_name is None assert ( - correlation_pearsonr_cube.long_name == "correlation_pearsonr_of_air_temperature" + correlation_pearsonr_cube.long_name == "Pearsonr_Correlation_of_air_temperature" ) @@ -75,7 +75,7 @@ def test_scores_additive_bias(cube: iris.cube.Cube): additive_bias_cube.data, np.zeros_like(additive_bias_cube.data), atol=1e-9 ) assert additive_bias_cube.standard_name is None - assert additive_bias_cube.long_name == "additive_bias_of_air_temperature" + assert additive_bias_cube.long_name == "Additive_Bias_of_air_temperature" def test_scores_mae(cube: iris.cube.Cube): From 7024b250ffe89e72fb8885acbc53f317eb2997f1 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 10:50:59 +0100 Subject: [PATCH 113/117] Typo in scores metrics testing of cube names #1759 --- tests/operators/test_scoreswrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index ceb42e5e0..179721ccd 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -55,7 +55,7 @@ def test_scores_correlation_pearsonr(cube: iris.cube.Cube): ) assert correlation_pearsonr_cube.standard_name is None assert ( - correlation_pearsonr_cube.long_name == "Pearsonr_Correlation_of_air_temperature" + correlation_pearsonr_cube.long_name == "Pearson_Correlation_of_air_temperature" ) From 611b7d4425f4f353f0743062cc65b96751fed08a Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 11:43:51 +0100 Subject: [PATCH 114/117] More detail in docstrings for scores methods in scoreswrappers.py #2214 --- src/CSET/operators/scoreswrappers.py | 111 ++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 3 deletions(-) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 876b41455..dad8ddc61 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -285,7 +285,42 @@ def scores_rmse(cubes: CubeList, preserved_coordinates: list[str] | str | None = def scores_mae(cubes: CubeList, preserved_coordinates: list[str] | str | None = None): - """Calculate the Mean Absolute Error (MAE) using scores.""" + r"""Calculate the Mean Absolute Error (MAE) using scores. + + Acts as a wrapper around the MAE calculation from ``scores`` ([scoresa]_, [scoresb]_). + + Parameters + ---------- + cubes: iris.cube.CubeList + A CubeList containing exactly two cubes: a base and an "other" model, + this can be an analysis and the model. + preserved_coordinates: list[str] | str | None, default is None. + The coordinates that you wish to preserve in the calculaiton of the + MAE. For example if you want a map of each time you can preserve + ["time","grid_latitude", "grid_longitude"] or if you want a time series + you can preserve ["time"], if you want to collapse to a single value + use `None`. The default is `None`. + + Returns + ------- + scores_cube: iris.cube.Cube + A cube containing the MAE between the base and other cube. + + References + ---------- + .. [scoresa] Leeuwenburg, T., Loveday, N., Ebert, E. E., Cook, H., + Khanarmuei, M., Taggart, R. J., Ramanathan, N., Carroll, M., Chong, S., + Griffiths, A., & Sharples, J. (2024) "scores: A Python package for + verifying and evaluating models and predictions with xarray". Journal + of Open Source Software, vol. 9, 6889. doi: 10.21105/joss.06889 + + .. [scoresb] Leeuwenburg, T., Loveday, N., Ramanathan, N., Chong, S., + Taggart, R. J., Shrestha, D., Khanarmuei, M., Cook, H., Bluett, L., Ebert, + E. E., Carroll, M., Trotta, B., Bishop, S., Squire, D. T., Griffiths, A., + Pagano, T. C., Fisher, A. J., Mandelbaum, T., Jinghan, F., … Smallwood, J. + (2026) "scores: Metrics for the verification, evaluation and optimisation of + forecasts, predictions or models (2.5.0)". Zenodo. doi: 10.5281/zenodo.18638494 + """ base, other = _sort_cubes_for_verification(cubes) # Copy the coordinates of the input cubes. @@ -338,7 +373,42 @@ def scores_mae(cubes: CubeList, preserved_coordinates: list[str] | str | None = def scores_additive_bias( cubes: CubeList, preserved_coordinates: list[str] | str | None = None ): - """Calculate the Additive Bias (Mean Error) using scores.""" + r"""Calculate the Additive Bias (Mean Error) using scores. + + Acts as a wrapper around the ME calculation from ``scores`` ([scoresa]_, [scoresb]_). + + Parameters + ---------- + cubes: iris.cube.CubeList + A CubeList containing exactly two cubes: a base and an "other" model, + this can be an analysis and the model. + preserved_coordinates: list[str] | str | None, default is None. + The coordinates that you wish to preserve in the calculaiton of the + ME. For example if you want a map of each time you can preserve + ["time","grid_latitude", "grid_longitude"] or if you want a time series + you can preserve ["time"], if you want to collapse to a single value + use `None`. The default is `None`. + + Returns + ------- + scores_cube: iris.cube.Cube + A cube containing the ME between the base and other cube. + + References + ---------- + .. [scoresa] Leeuwenburg, T., Loveday, N., Ebert, E. E., Cook, H., + Khanarmuei, M., Taggart, R. J., Ramanathan, N., Carroll, M., Chong, S., + Griffiths, A., & Sharples, J. (2024) "scores: A Python package for + verifying and evaluating models and predictions with xarray". Journal + of Open Source Software, vol. 9, 6889. doi: 10.21105/joss.06889 + + .. [scoresb] Leeuwenburg, T., Loveday, N., Ramanathan, N., Chong, S., + Taggart, R. J., Shrestha, D., Khanarmuei, M., Cook, H., Bluett, L., Ebert, + E. E., Carroll, M., Trotta, B., Bishop, S., Squire, D. T., Griffiths, A., + Pagano, T. C., Fisher, A. J., Mandelbaum, T., Jinghan, F., … Smallwood, J. + (2026) "scores: Metrics for the verification, evaluation and optimisation of + forecasts, predictions or models (2.5.0)". Zenodo. doi: 10.5281/zenodo.18638494 + """ base, other = _sort_cubes_for_verification(cubes) # Copy the coordinates of the input cubes. @@ -390,7 +460,42 @@ def scores_additive_bias( def scores_correlation_pearsonr( cubes: CubeList, preserved_coordinates: list[str] | str | None = None ): - """Calculate the Pearson's Correlation using scores.""" + r"""Calculate the Pearson's Correlation (PC) coefficient using scores. + + Acts as a wrapper around the PC calculation from ``scores`` ([scoresa]_, [scoresb]_). + + Parameters + ---------- + cubes: iris.cube.CubeList + A CubeList containing exactly two cubes: a base and an "other" model, + this can be an analysis and the model. + preserved_coordinates: list[str] | str | None, default is None. + The coordinates that you wish to preserve in the calculaiton of the + PC. For example if you want a map of each time you can preserve + ["time","grid_latitude", "grid_longitude"] or if you want a time series + you can preserve ["time"], if you want to collapse to a single value + use `None`. The default is `None`. + + Returns + ------- + scores_cube: iris.cube.Cube + A cube containing the PC between the base and other cube. + + References + ---------- + .. [scoresa] Leeuwenburg, T., Loveday, N., Ebert, E. E., Cook, H., + Khanarmuei, M., Taggart, R. J., Ramanathan, N., Carroll, M., Chong, S., + Griffiths, A., & Sharples, J. (2024) "scores: A Python package for + verifying and evaluating models and predictions with xarray". Journal + of Open Source Software, vol. 9, 6889. doi: 10.21105/joss.06889 + + .. [scoresb] Leeuwenburg, T., Loveday, N., Ramanathan, N., Chong, S., + Taggart, R. J., Shrestha, D., Khanarmuei, M., Cook, H., Bluett, L., Ebert, + E. E., Carroll, M., Trotta, B., Bishop, S., Squire, D. T., Griffiths, A., + Pagano, T. C., Fisher, A. J., Mandelbaum, T., Jinghan, F., … Smallwood, J. + (2026) "scores: Metrics for the verification, evaluation and optimisation of + forecasts, predictions or models (2.5.0)". Zenodo. doi: 10.5281/zenodo.18638494 + """ base, other = _sort_cubes_for_verification(cubes) # Copy the coordinates of the input cubes. From 1106bb138eb0a9bed67c6001c0fe05b031134eb7 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 12:09:19 +0100 Subject: [PATCH 115/117] Removed input cubes from test test_scores_rmse_nonzero #2214 --- .../verification/surface_rmse_scores.yaml | 45 ------------------- tests/operators/test_scoreswrappers.py | 2 +- 2 files changed, 1 insertion(+), 46 deletions(-) delete mode 100644 src/CSET/recipes/verification/surface_rmse_scores.yaml diff --git a/src/CSET/recipes/verification/surface_rmse_scores.yaml b/src/CSET/recipes/verification/surface_rmse_scores.yaml deleted file mode 100644 index 5e834ea94..000000000 --- a/src/CSET/recipes/verification/surface_rmse_scores.yaml +++ /dev/null @@ -1,45 +0,0 @@ -category: Scores -title: "$METHOD $VARNAME\nRMSE between $OTHER_MODEL and $BASE_MODEL" -description: | - Extracts and plots the Root Mean Square Error in $METHOD of $VARNAME - for all times. The RMSE is calculated based on that used in the - package [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.continuous.rmse). This recipe preserves the time, latitude and longitude coordinates. - This means a sequence of spatial plots of the RMSE can be produced. - - A larger RMSE implies a greater error than a smaller RMSE. An - RMSE of zero implies the two field match. The RMSE is calculated - on the grid point and thus a spatial view of the RMSE provides useful - information about where the differences are, or if placement errors - are domininating the score (usually indicated by dipoles). - -steps: - - operator: read.read_cubes - file_paths: $INPUT_PATHS - model_names: [$BASE_MODEL, $OTHER_MODEL] - constraint: - operator: constraints.combine_constraints - varname_constraint: - operator: constraints.generate_var_constraint - varname: $VARNAME - cell_methods_constraint: - operator: constraints.generate_cell_methods_constraint - cell_methods: [] - varname: $VARNAME - pressure_level_constraint: - operator: constraints.generate_level_constraint - coordinate: "pressure" - levels: [] - subarea_type: $SUBAREA_TYPE - subarea_extent: $SUBAREA_EXTENT - - - operator: scoreswrappers.scores_rmse - preserved_coordinates: ["time", "grid_latitude", "grid_longitude"] - - - operator: collapse.collapse - coordinate: ["time"] - method: $METHOD - - - operator: plot.spatial_pcolormesh_plot - - - operator: write.write_cube_to_nc - overwrite: True diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 179721ccd..725b6a95a 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -112,7 +112,7 @@ def test_scores_rmse(cube: iris.cube.Cube): assert rmse_cube.long_name == "RMSE_of_air_temperature" -def test_scores_rmse_nonzero(cube: iris.cube.Cube): +def test_scores_rmse_nonzero(): """Test taking the rmse between two different cubes.""" # Data preparation. cube = iris.cube.Cube( From e3f7b7cc80ee6cb7c299f8864de1c330e6daaa78 Mon Sep 17 00:00:00 2001 From: Bernard Claxton Date: Fri, 17 Jul 2026 12:19:45 +0100 Subject: [PATCH 116/117] Minor typo in description of recipe corrected #2214 --- .../verification/timeseries_surface_difference_scores_MAE.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml index f58ac8d6d..d5973fbbb 100644 --- a/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml +++ b/src/CSET/recipes/verification/timeseries_surface_difference_scores_MAE.yaml @@ -8,7 +8,7 @@ description: | collapsed over all other coordinates in the cube and calculated for every timestep. A larger MAE implies a greater error than a smaller MAE. An - RMSE of zero indicates the two fields most likely match. The MAE is calculated + MAE of zero indicates the two fields most likely match. The MAE is calculated on the grid point and thus a spatial view of the MAE provides useful information about where the differences are, or if placement errors are domininating the score (usually indicated by dipoles). MAE is a fair From 630e5269b9e375071e8df37f9c3816404c396246 Mon Sep 17 00:00:00 2001 From: BernardClaxton Date: Fri, 17 Jul 2026 14:21:34 +0100 Subject: [PATCH 117/117] Apply suggestion from @jfrost-mo Co-authored-by: James Frost --- tests/operators/test_scoreswrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 725b6a95a..1f035c3b1 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -49,8 +49,8 @@ def test_scores_correlation_pearsonr(cube: iris.cube.Cube): # As both cubes use the same data, check the Pearson correlation is one. assert isinstance(correlation_pearsonr_cube, iris.cube.Cube) assert np.allclose( - correlation_pearsonr_cube.data - 1.0, - np.zeros_like(correlation_pearsonr_cube.data), + correlation_pearsonr_cube.data, + np.ones_like(correlation_pearsonr_cube.data), atol=1e-9, ) assert correlation_pearsonr_cube.standard_name is None