Add Scores diagnostics for spatial and timeseries RMSE, Additive Bias, MAE, and Pearson’s Correlation Coefficient

src/CSET/cset_workflow/meta/verification/rose-meta.conf

@@ -53,38 +53,140 @@ sort-key=met5
 
 
 ###################################
-# Scores
-[Verification/Scores]
-ns=Verification/Scores
-sort-key=scores
-title=Scores
 
 [template variables=SCORES_ALL]
-ns=Verification/Scores
+ns=Verification
 description=If selected, all scores shown below will be calculated.
             NOTE scores currently only available for model-model intercomparisons.
 type=python_boolean
-trigger=template variables=SCORES_RMSE_SPATIAL: False;
-        template variables=SCORES_RMSE_TIMESERIES: False;
+trigger=template variables=SCORES_SPATIAL_RMSE: False;
+        template variables=SCORES_SPATIAL_MAE: False;
+        template variables=SCORES_SPATIAL_AB: False;
+        template variables=SCORES_TIMESERIES_RMSE: False;
+        template variables=SCORES_TIMESERIES_MAE: False;
+        template variables=SCORES_TIMESERIES_AB: False;
+        template variables=SCORES_TIMESERIES_PC: False;
 compulsory=true
 sort-key=scores1
 
-[template variables=SCORES_RMSE_SPATIAL]
-ns=Verification/Scores
-description=Create spatially mapped RMSE plots for the specified surface fields.
-            The analysis method(s) set in SPATIAL_SURFACE_FIELD_METHOD will be used.
-            NOTE currently only available for model-model intercomparisons.
+
+# Scores spatial plots
+[Verification/ScoresSpatial]
+ns=Verification/ScoresSpatial
+sort-key=scoresspatial
+title=Scores spatial
+
+[template variables=SPATIAL_SCORES_FIELD_METHOD]
+ns=Verification/ScoresSpatial
+description=Select analysis method(s) for output mapped plots. Add all options required.
+            Leave blank or set to "SEQ" (sequence) for plots each diagnostic output time.
+            For time-collapsed outputs over each analysis period, set to "CASE", "MAX", "MIN" etc.
+            Selecting "CASE" will compute the spatial scores metrics over the entire case study.
+help=Quoted analysis methods. Settings should be based on available iris.analysis methods for collapsing cube dimensions.
+type=python_list
+compulsory=true
+sort-key=sc-sp-0
+
+[template variables=SCORES_SPATIAL_RMSE]
+ns=Verification/ScoresSpatial
+title=Root Mean Square Error
+description=SCORES COUNTINOUS METRIC.
+            Plots a 2D spatial plot of the RMSE between two models.
+            The computation is implemented by the scores package.
+            For the time coordinate, the method(s) set in
+            SPATIAL_SCORES_FIELD_METHOD will be used.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
 type=python_boolean
 compulsory=true
-sort-key=scores2
+sort-key=sc-sp-1
 
-[template variables=SCORES_RMSE_TIMESERIES]
-ns=Verification/Scores
-description=Create a timeseries of the RMSE for the specified surface fields.
-            NOTE currently only available for model-model intercomparisons.
+[template variables=SCORES_SPATIAL_MAE]
+ns=Verification/ScoresSpatial
+title=Mean Absolute Error
+description=SCORES COUNTINOUS METRIC.
+            Plots a 2D spatial plot of the MAE between two models.
+            The computation is implemented by the scores package.
+            For the time coordinate, the method(s) set in
+            SPATIAL_SCORES_FIELD_METHOD will be used.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
+type=python_boolean
+compulsory=true
+sort-key=sc-sp-2
+
+[template variables=SCORES_SPATIAL_AB]
+ns=Verification/ScoresSpatial
+title=Additive Bias (Mean Error)
+description=SCORES COUNTINOUS METRIC.
+            Plots a 2D spatial plot of the Additive Bias between two models.
+            The Additive Bias is also known as the Mean Error.
+            The computation is implemented by the scores package.
+            For the time coordinate, the method(s) set in
+            SPATIAL_SCORES_FIELD_METHOD will be used.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
+type=python_boolean
+compulsory=true
+sort-key=sc-sp-3
+
+
+###################################
+
+# Scores timeseries plots
+[Verification/ScoresTimeseries]
+ns=Verification/ScoresTimeseries
+sort-key=scorestimeseries
+title=Scores timeseries
+
+[template variables=SCORES_TIMESERIES_RMSE]
+ns=Verification/ScoresTimeseries
+title=Root Mean Square Error
+description=SCORES COUNTINOUS METRIC.
+            Plots a timeseries plot of the RMSE between two models.
+            The computation is implemented by the scores package.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
+type=python_boolean
+compulsory=true
+sort-key=sc-ts-1
+
+[template variables=SCORES_TIMESERIES_MAE]
+ns=Verification/ScoresTimeseries
+title=Mean Absolute Error
+description=SCORES COUNTINOUS METRIC.
+            Plots a timeseries plot of the MAE between two models.
+            The computation is implemented by the scores package.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
+type=python_boolean
+compulsory=true
+sort-key=sc-ts-2
+
+[template variables=SCORES_TIMESERIES_AB]
+ns=Verification/ScoresTimeseries
+title=Additive Bias (Mean Error)
+description=SCORES COUNTINOUS METRIC.
+            Plots a timeseries plot of the Additive Bias between two models.
+            The Additive Bias is also known as the Mean Error.
+            The computation is implemented by the scores package.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
+type=python_boolean
+compulsory=true
+sort-key=sc-ts-3
+
+[template variables=SCORES_TIMESERIES_PC]
+ns=Verification/ScoresTimeseries
+title=Pearson's Correlation
+description=SCORES COUNTINOUS METRIC.
+            Plots a timeseries plot of the Pearson’s Correlation Coefficient between two models.
+            The computation is implemented by the scores package.
+help=This computes the metric using the scores package. Details of this can
+     be found at https://scores.readthedocs.io/en/stable/included.html#continuous
 type=python_boolean
 compulsory=true
-sort-key=scores3
+sort-key=sc-ts-4
 
 [template variables=SCORES_CRPS_FOR_ENSEMBLE]
 ns=Verification/Scores

src/CSET/loaders/verification.py

@@ -19,24 +19,76 @@
 from CSET.recipes import Config, RawRecipe, get_models
 
 
+def _get_scores_spatial_methods(conf):
+    """Compile list of the required scores spatial plots."""
+    scores_spatial_methods = []
+    if conf.SCORES_SPATIAL_RMSE or conf.SCORES_ALL:
+        scores_spatial_methods.append("RMSE")
+    if conf.SCORES_SPATIAL_AB or conf.SCORES_ALL:
+        scores_spatial_methods.append("additive_bias")
+    if conf.SCORES_SPATIAL_MAE or conf.SCORES_ALL:
+        scores_spatial_methods.append("MAE")
+    return scores_spatial_methods
+
+
+def _get_scores_timeseries_methods(conf):
+    """Compile list of the required scores timeseries plots."""
+    scores_timeseries_methods = []
+    if conf.SCORES_TIMESERIES_RMSE or conf.SCORES_ALL:
+        scores_timeseries_methods.append("RMSE")
+    if conf.SCORES_TIMESERIES_AB or conf.SCORES_ALL:
+        scores_timeseries_methods.append("additive_bias")
+    if conf.SCORES_TIMESERIES_MAE or conf.SCORES_ALL:
+        scores_timeseries_methods.append("MAE")
+    if conf.SCORES_TIMESERIES_PC or conf.SCORES_ALL:
+        scores_timeseries_methods.append("correlation_pearsonr")
+    return scores_timeseries_methods
+
+
 def load(conf: Config):
     """Yield recipes from the given workflow configuration."""
     # Load a list of model detail dictionaries.
     models = get_models(conf.asdict())
     # Models are listed in order, so model 1 is the first element.
 
-    if conf.SCORES_RMSE_SPATIAL or conf.SCORES_ALL:
+    scores_spatial_methods = _get_scores_spatial_methods(conf)
+    if scores_spatial_methods:
+        # Produce 2D spatial plots of scores metrics.
         base_model = models[0]
-        for model, field, method in itertools.product(
-            models[1:], conf.SURFACE_FIELDS, conf.SPATIAL_SURFACE_FIELD_METHOD
+        for model, field, method, scores_method in itertools.product(
+            models[1:],
+            conf.SURFACE_FIELDS,
+            conf.SPATIAL_SCORES_FIELD_METHOD,
+            scores_spatial_methods,
         ):
+            preserved_coords = ["time", "grid_latitude", "grid_longitude"]
+            method_null = ""
+            scores_method_case = "CASE"
+            scores_coords_case = ["grid_latitude", "grid_longitude"]
+            if scores_method == "RMSE" and method == scores_method_case:
+                # Set the preserved coords and collapse method required
+                # to produce RMSE spatial plot over an entire case study.
+                preserved_coords = scores_coords_case
+                method = method_null
+            if scores_method == "MAE" and method == scores_method_case:
+                # Set the preserved coords and collapse method required
+                # to produce MAE spatial plot over an entire case study.
+                preserved_coords = scores_coords_case
+                method = method_null
+            if scores_method == "additive_bias" and method == scores_method_case:
+                # Set the preserved coords and collapse method required
+                # to produce ME additive bias spatial plot over an entire case study.
+                preserved_coords = scores_coords_case
+                method = method_null
             yield RawRecipe(
-                recipe="surface_rmse_scores.yaml",
+                recipe=f"surface_difference_scores_{scores_method}.yaml",
                 variables={
                     "VARNAME": field,
                     "BASE_MODEL": base_model["name"],
                     "OTHER_MODEL": model["name"],
                     "METHOD": method,
+                    "PRESERVED_COORDS": preserved_coords,
+                    "SUBAREA_NAME": conf.SUBAREA_NAME if conf.SELECT_SUBAREA else "",
                     "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None,
                     "SUBAREA_EXTENT": conf.SUBAREA_EXTENT
                     if conf.SELECT_SUBAREA
@@ -46,15 +98,20 @@ def load(conf: Config):
                 aggregation=False,
             )
 
-    if conf.SCORES_RMSE_TIMESERIES or conf.SCORES_ALL:
+    scores_timeseries_methods = _get_scores_timeseries_methods(conf)
+    if scores_timeseries_methods:
+        # Produce timeseries plots of scores metrics averaged over the domain for each case study.
         base_model = models[0]
-        for model, field in itertools.product(models[1:], conf.SURFACE_FIELDS):
+        for model, field, scores_method in itertools.product(
+            models[1:], conf.SURFACE_FIELDS, scores_timeseries_methods
+        ):
             yield RawRecipe(
-                recipe="timeseries_surface_rmse_scores.yaml",
+                recipe=f"timeseries_surface_difference_scores_{scores_method}.yaml",
                 variables={
                     "VARNAME": field,
                     "BASE_MODEL": base_model["name"],
                     "OTHER_MODEL": model["name"],
+                    "SUBAREA_NAME": conf.SUBAREA_NAME if conf.SELECT_SUBAREA else "",
                     "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None,
                     "SUBAREA_EXTENT": conf.SUBAREA_EXTENT
                     if conf.SELECT_SUBAREA