From a4365b9f9818f1b6fbae94b6c4900bd3b5c0df35 Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 09:52:53 +0100 Subject: [PATCH 01/17] add ets section to gui --- .../meta/verification/rose-meta.conf | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 561ee16fb..2e2dc98d4 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -188,7 +188,7 @@ sort-key=sc-ts-3 [template variables=SCORES_TIMESERIES_PC] ns=Verification/ScoresTimeseries -title=Pearson's Correlation +title=Pearsons Correlation description=SCORES COUNTINOUS METRIC. Plots a timeseries plot of the Pearson’s Correlation Coefficient between two models. The computation is implemented by the scores package. @@ -320,15 +320,10 @@ type=python_boolean compulsory=true sort-key=sc-sp-1_model_vs_obs -[Verification/Scores Categorical] -[template variables=SCORES_CATEGORICAL_POD] -ns=Verification/Scores Categorical -description=Compute Scores Probability of Detection. -type=python_boolean -compulsory=true -trigger=template variables=SCORES_CATEGORICAL_POD_ENTRIES: True -sort-key=scoresPOD1 +##### + +# Categorical Scores metrics against observations. [template variables=SCORES_CATEGORICAL_POD_ENTRIES] ns=Verification/Scores Categorical description=List of variables, operators and thresholds @@ -343,4 +338,22 @@ help=Supported operators: ['air_temperature,lt,280'] type=python_list compulsory=true +sort-key=scoresPOD1 + +[Verification/Scores Categorical] +[template variables=SCORES_CATEGORICAL_POD] +ns=Verification/Scores Categorical +description=Compute Scores Probability of Detection. +type=python_boolean +compulsory=true +trigger=template variables=SCORES_CATEGORICAL_POD_ENTRIES: True sort-key=scoresPOD2 + +[Verification/Scores Categorical] +[template variables=SCORES_CATEGORICAL_ETS] +ns=Verification/Scores Categorical +description=Compute Scores Equitable Threat Score. +type=python_boolean +compulsory=true +trigger=template variables=SCORES_CATEGORICAL_ETS_ENTRIES: True +sort-key=scoresPOD3 From d1c4b15f57af3834a7a1a548dfa8a8aa41151491 Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 10:59:25 +0100 Subject: [PATCH 02/17] add ets to rose-suite.conf and revert pod default to false --- src/CSET/cset_workflow/rose-suite.conf.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index 2880435e5..afc39cc55 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -128,7 +128,8 @@ POINT_OBS=False !!POINT_OBS_USE_WMO_STATION_NUMBERS=False !!POINT_OBS_WMO_BLOCK_STTN_NUMBERS=[] PRESSURE_LEVELS=[] -SCORES_CATEGORICAL_POD=True +SCORES_CATEGORICAL_ETS=False +SCORES_CATEGORICAL_POD=False SCORES_CATEGORICAL_POD_ENTRIES=[""] PRESSURE_LEVEL_FIELDS=[] !!PROB_TEMPERATURE_CONDITION=[] From adb176e96b5fa26b1351c3aef1907afcd799fefe Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 11:13:35 +0100 Subject: [PATCH 03/17] generalise --- src/CSET/cset_workflow/rose-suite.conf.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/rose-suite.conf.example b/src/CSET/cset_workflow/rose-suite.conf.example index afc39cc55..ec783a0b7 100644 --- a/src/CSET/cset_workflow/rose-suite.conf.example +++ b/src/CSET/cset_workflow/rose-suite.conf.example @@ -130,7 +130,7 @@ POINT_OBS=False PRESSURE_LEVELS=[] SCORES_CATEGORICAL_ETS=False SCORES_CATEGORICAL_POD=False -SCORES_CATEGORICAL_POD_ENTRIES=[""] +SCORES_CATEGORICAL_ENTRIES=[""] PRESSURE_LEVEL_FIELDS=[] !!PROB_TEMPERATURE_CONDITION=[] !!PROB_TEMPERATURE_THRESHOLD=[] From 342faccce83e067e2745575311e6bdadf4163c5e Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 11:16:07 +0100 Subject: [PATCH 04/17] add new ets recipe --- .../surface_categorical_model_obs_ets.yaml | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml diff --git a/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml new file mode 100644 index 000000000..b9a35a453 --- /dev/null +++ b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml @@ -0,0 +1,76 @@ +category: Scores Categorical +title: Timeseries of ETS for "$VARNAME over $SUBAREA_NAME, using threshold $OPERATOR $THRESHOLD" +description: | + + The Equitable Threat Score (ETS) evaluates the accuracy of forecasts for events that meet a specified threshold, + hile accounting for correct forecasts that could occur purely by chance. Unlike the Probability of Detection (POD), + ETS considers hits, misses, and false alarms, providing a more balanced assessment of forecast skill. + + For example, if the threshold is 290 K and op_func is gt (greater than), an observation of 292 K and a forecast of 295 K + would be counted as a hit. ETS adjusts the total number of hits by removing the number of hits expected due to random chance. + + It is calculated as: + + .. math:: + + ETS = \frac{hits - hits_{random}} + {hits + misses + false\ alarms - hits_{random}} + + ETS ranges from -1/3 to 1, where 1 indicates a perfect forecast, 0 indicates no skill beyond random chance, and negative values indicate worse than + random chance. + + References + https://scores.readthedocs.io/en/stable/api.html#scores.categorical.BasicContingencyManager.equitable_threat_score + +steps: + - operator: read.read_cubes + file_paths: $INPUT_PATHS + model_names: $MODEL_NAME + constraint: + operator: constraints.combine_constraints + varname_constraint: + operator: constraints.generate_var_constraint + varname: ['observed_$VARNAME', '$VARNAME'] + cell_methods_constraint: + operator: constraints.generate_cell_methods_constraint + cell_methods: [] + pressure_level_constraint: + operator: constraints.generate_level_constraint + coordinate: "pressure" + levels: [] + subarea_type: $SUBAREA_TYPE + subarea_extent: $SUBAREA_EXTENT + + - operator: misc.extract_common_points + coordinate: time + + - operator: misc.combine_cubes_into_cubelist + first: + operator: filters.filter_cubes + constraint: + operator: constraints.generate_var_constraint + varname: observed_$VARNAME + second: + operator: regrid.interpolate_to_point_cube + fld: + operator: filters.filter_multiple_cubes + constraint: + operator: constraints.combine_constraints + var_constraint: + operator: constraints.generate_var_constraint + varname: $VARNAME + point_cube: + operator: filters.filter_cubes + constraint: + operator: constraints.generate_var_constraint + varname: observed_$VARNAME + + - operator: scoreswrappers.scores_ets_model_obs + preserved_coordinates: "time" + threshold: $THRESHOLD + op_func: $OPERATOR + + - operator: plot.plot_line_series + + - operator: write.write_cube_to_nc + overwrite: True From 6a4d8fb0027aea9ca5c49a49b9ba0589ba0e1513 Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 11:17:32 +0100 Subject: [PATCH 05/17] generalise categorical operaters/vars --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 2e2dc98d4..6cb9d3d66 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -324,7 +324,7 @@ sort-key=sc-sp-1_model_vs_obs ##### # Categorical Scores metrics against observations. -[template variables=SCORES_CATEGORICAL_POD_ENTRIES] +[template variables=SCORES_CATEGORICAL_ENTRIES] ns=Verification/Scores Categorical description=List of variables, operators and thresholds i.e. [',,'] From 832174881479bb7fef30b4151117a0264754d196 Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 11:19:37 +0100 Subject: [PATCH 06/17] add new entry for ets --- src/CSET/loaders/verification.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index c0b136a30..8329c5dbf 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -66,6 +66,8 @@ def _get_scores_timeseries_categorical(conf): scores_timeseries_categorical = [] if conf.SCORES_CATEGORICAL_POD or conf.SCORES_ALL: scores_timeseries_categorical.append("pod") + if conf.SCORES_CATEGORICAL_ETS or conf.SCORES_ALL: + scores_timeseries_categorical.append("ets") return scores_timeseries_categorical @@ -288,7 +290,7 @@ def load(conf: Config): if scores_timeseries_categorical: # Produce timeseries plots of scores categorical metrics for each model. for field_and_method, scores_method in itertools.product( - conf.SCORES_CATEGORICAL_POD_ENTRIES, scores_timeseries_categorical + conf.SCORES_CATEGORICAL_ENTRIES, scores_timeseries_categorical ): var, op, value = field_and_method.split(",") From a558709f12d3e4ef2c8ac33835fc35b4e811359f Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 11:26:15 +0100 Subject: [PATCH 07/17] add ets operator --- src/CSET/operators/scoreswrappers.py | 107 +++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 681f7f5be..8373c8f43 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -717,3 +717,110 @@ def scores_pod_model_obs( scores_results.append(scores_cube) return scores_results + + +def scores_ets_model_obs( + cubes: CubeList, + preserved_coordinates: list[str] | str | None, + threshold: str, + op_func: str, +): + r""" + Compute the Equitable Threat Score (ETS) score using Scores ([scoresa]_ [scoresb]_). + + Parameters + ---------- + cubes: iris.cube.CubeList + An iris cubelist containing model(s) and an observation cube. + preserved_coordinates: list | str | None + An object containing which coordinates to preserve in the computation. For example, if cubes contain shape time, point location, + then preserving coordinate 'time' will produce the equitable threat score for each timeslice (shape time). If None, + then it will return a single value score for all times/point locations. + threshold: str + A str containing the threshold to use to generate the binary masks, which subsequently gets turned to a float (but passed as str around the recipe templating). + op_func: str + A string either containing 'lt' for less than or 'gt for greater than, to determine how the threshold is applied to the data + to generate the mask. + + Returns + ------- + cube: iris.cube + An iris cube, containing the probability of detection score for further plotting. + + Notes + ----- + The Equitable Threat Score (ETS) evaluates the accuracy of forecasts for events that meet a specified threshold, + hile accounting for correct forecasts that could occur purely by chance. Unlike the Probability of Detection (POD), + ETS considers hits, misses, and false alarms, providing a more balanced assessment of forecast skill. + + For example, if the threshold is 290 K and op_func is gt (greater than), an observation of 292 K and a forecast of 295 K + would be counted as a hit. ETS adjusts the total number of hits by removing the number of hits expected due to random chance. + + It is calculated as: + + .. math:: + + ETS = \frac{hits - hits_{random}} + {hits + misses + false\ alarms - hits_{random}} + + ETS ranges from -1/3 to 1, where 1 indicates a perfect forecast, 0 indicates no skill beyond random chance, and negative values indicate worse than + random chance. + """ + # Split out model(s) and obs + models = CubeList() + for c in cubes: + if "observed" in c.long_name: + observed = c + else: + models.append(c) + + # Setup cubelist to store results + scores_results = iris.cube.CubeList() + + # Setup operators greater than, less than. + ops = { + "gt": operator.gt, + "lt": operator.lt, + } + + try: + op = ops[op_func] + except KeyError as err: + raise ValueError(f"Operator {op_func} not supported.") from err + + for model in models: + # Convert obs cubes to xarray and resolve preserved dimensions. + other_xr = xr.DataArray.from_iris(model) + base_xr = xr.DataArray.from_iris(observed) + preserve_dims = _resolve_preserve_dims( + observed, other_xr, preserved_coordinates + ) + + # Create event operator object using threshold and operator direction. + event_operator = scores.categorical.ThresholdEventOperator( + default_event_threshold=float(threshold), default_op_fn=op + ) + + # Generate binary fields using the event operator. + forecast_binary, observed_binary = event_operator.make_event_tables( + other_xr, base_xr + ) + + # Create binary contigency manager, as per Scores API, using transform to preserve preserve_dims + contingency_manager = scores.categorical.BinaryContingencyManager( + forecast_binary, observed_binary + ).transform(preserve_dims=preserve_dims) + + # Get ETS from the contigency manager, and convert back to an iris cube. + scores_cube = xr.DataArray.to_iris(contingency_manager.equitable_threat_score()) + + # Rename cube so it plots correctly alongside correcting cube units. + scores_cube.rename( + f"Equitable_Threat_Score_{op_func}_{threshold}_{observed.name()}" + ) + scores_cube.units = "1" + scores_cube.attributes["model_name"] = model.attributes["model_name"] + + scores_results.append(scores_cube) + + return scores_results From 3552d9f58ed3074c389d0a12d189fb6e7b9c2dbd Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 11:26:55 +0100 Subject: [PATCH 08/17] generalised operator/threshold --- .../verification/surface_categorical_model_obs_pod.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml b/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml index 1249caa11..6791f2b7e 100644 --- a/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml +++ b/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml @@ -1,5 +1,5 @@ category: Scores Categorical -title: Timeseries of POD for "$VARNAME over $SUBAREA_NAME, using threshold $POD_OPERATOR $POD_THRESHOLD" +title: Timeseries of POD for "$VARNAME over $SUBAREA_NAME, using threshold $OPERATOR $THRESHOLD" description: | Create a timeseries of the probability of detection. @@ -62,8 +62,8 @@ steps: - operator: scoreswrappers.scores_pod_model_obs preserved_coordinates: "time" - threshold: $POD_THRESHOLD - op_func: $POD_OPERATOR + threshold: $THRESHOLD + op_func: $OPERATOR - operator: plot.plot_line_series From 3a9d8986e44f901fb8f8ad26a4c8da9f3d9ed4b5 Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 14:24:48 +0100 Subject: [PATCH 09/17] add extra description --- src/CSET/operators/scoreswrappers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/CSET/operators/scoreswrappers.py b/src/CSET/operators/scoreswrappers.py index 8373c8f43..46a1326e9 100644 --- a/src/CSET/operators/scoreswrappers.py +++ b/src/CSET/operators/scoreswrappers.py @@ -763,6 +763,10 @@ def scores_ets_model_obs( ETS = \frac{hits - hits_{random}} {hits + misses + false\ alarms - hits_{random}} + where + + hits_{random} = \frac{(hits + misses)(hits + false\ alarms)}{total count} + ETS ranges from -1/3 to 1, where 1 indicates a perfect forecast, 0 indicates no skill beyond random chance, and negative values indicate worse than random chance. """ From 3de0ecd9471fed84bd2637440f64891bf6d031ba Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 14:25:44 +0100 Subject: [PATCH 10/17] add additional text --- .../verification/surface_categorical_model_obs_ets.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml index b9a35a453..a8f3f3c40 100644 --- a/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml +++ b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml @@ -16,6 +16,11 @@ description: | ETS = \frac{hits - hits_{random}} {hits + misses + false\ alarms - hits_{random}} + + where + + hits_{random} = \frac{(hits + misses)(hits + false\ alarms)}{total count} + ETS ranges from -1/3 to 1, where 1 indicates a perfect forecast, 0 indicates no skill beyond random chance, and negative values indicate worse than random chance. From b64d895ed536e5bc96203782646a13549ba29f54 Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 14:26:48 +0100 Subject: [PATCH 11/17] add provisional tests --- tests/operators/test_scoreswrappers.py | 123 +++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index a4430a27a..36f413856 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -540,3 +540,126 @@ def test_invalid_operator_raises(make_cube_categorical_testing): threshold="10", op_func="gte", ) + + +def test_ets_gt_perfect_forecast(make_cube_categorical_testing): + """Perfect forecast should give ETS=1.""" + obs = make_cube_categorical_testing( + [[12, 5], [15, 8]], + long_name="observed_temperature", + model_name="obs", + ) + obs.attributes["cset_comparison_base"] = 1 + + model = make_cube_categorical_testing( + [[12, 5], [15, 8]], + long_name="temperature", + model_name="test_model", + ) + + result = scoreswrappers.scores_ets_model_obs( + CubeList([model, obs]), + preserved_coordinates=None, + threshold="10", + op_func="gt", + ) + + assert len(result) == 1 + assert np.allclose(result[0].data, 1.0) + + +def test_ets_gt_mixed_case(make_cube_categorical_testing): + """Manual ETS calculation for a mixed forecast.""" + obs = make_cube_categorical_testing( + [[12, 5], [15, 8]], + long_name="observed_temperature", + model_name="obs", + ) + obs.attributes["cset_comparison_base"] = 1 + + model = make_cube_categorical_testing( + [[14, 20], [7, 4]], + long_name="temperature", + model_name="test_model", + ) + + result = scoreswrappers.scores_ets_model_obs( + CubeList([model, obs]), + preserved_coordinates=None, + threshold="10", + op_func="gt", + ) + + # Binary fields: + # + # Obs = [[1,0], + # [1,0]] + # + # Model = [[1,1], + # [0,0]] + # + # H=1, M=1, F=1, N=4 + # Hr=(2*2)/4=1 + # ETS=(1-1)/(1+1+1-1)=0 + + assert len(result) == 1 + assert np.allclose(result[0].data, 0.0) + + +def test_ets_gt_complete_miss(make_cube_categorical_testing): + """No hits, all events misplaced.""" + obs = make_cube_categorical_testing( + [[12, 12], [5, 5]], + long_name="observed_temperature", + model_name="obs", + ) + obs.attributes["cset_comparison_base"] = 1 + + model = make_cube_categorical_testing( + [[5, 5], [12, 12]], + long_name="temperature", + model_name="test_model", + ) + + result = scoreswrappers.scores_ets_model_obs( + CubeList([model, obs]), + preserved_coordinates=None, + threshold="10", + op_func="gt", + ) + + # H=0, M=2, F=2, N=4 + # Hr=(2*2)/4=1 + # ETS=(0-1)/(0+2+2-1)=-1/3 + + assert len(result) == 1 + assert np.allclose(result[0].data, -1.0 / 3.0) + + +def test_ets_metadata(make_cube_categorical_testing): + """Check output cube metadata.""" + obs = make_cube_categorical_testing( + [[12, 5], [15, 8]], + long_name="observed_temperature", + model_name="obs", + ) + obs.attributes["cset_comparison_base"] = 1 + + model = make_cube_categorical_testing( + [[12, 5], [15, 8]], + long_name="temperature", + model_name="ukv", + ) + + result = scoreswrappers.scores_ets_model_obs( + CubeList([model, obs]), + preserved_coordinates=None, + threshold="10", + op_func="gt", + ) + + cube = result[0] + + assert cube.units == "1" + assert cube.attributes["model_name"] == "ukv" + assert cube.name() == "Equitable_Threat_Score_gt_10_observed_temperature" From 8653f369974e2089fb9863ba0376104d18924cda Mon Sep 17 00:00:00 2001 From: James Warner Date: Mon, 17 Aug 2026 16:33:45 +0100 Subject: [PATCH 12/17] correct arg vals --- src/CSET/loaders/verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/CSET/loaders/verification.py b/src/CSET/loaders/verification.py index 8329c5dbf..84c24adec 100644 --- a/src/CSET/loaders/verification.py +++ b/src/CSET/loaders/verification.py @@ -299,8 +299,8 @@ def load(conf: Config): variables={ "VARNAME": var, "MODEL_NAME": ["OBS"] + [model["name"] for model in models], - "POD_THRESHOLD": value, - "POD_OPERATOR": op, + "THRESHOLD": value, + "OPERATOR": op, "SUBAREA_NAME": conf.SUBAREA_NAME if conf.SELECT_SUBAREA else "", "SUBAREA_TYPE": conf.SUBAREA_TYPE if conf.SELECT_SUBAREA else None, "SUBAREA_EXTENT": conf.SUBAREA_EXTENT From bba8a1769251d110b727f8232e9865c0fba780c2 Mon Sep 17 00:00:00 2001 From: James Warner Date: Wed, 19 Aug 2026 09:41:09 +0100 Subject: [PATCH 13/17] remove backslash that breaks formatting --- .../verification/surface_categorical_model_obs_ets.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml index a8f3f3c40..4ee74cb92 100644 --- a/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml +++ b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml @@ -11,15 +11,13 @@ description: | It is calculated as: - .. math:: - - ETS = \frac{hits - hits_{random}} + ETS = frac{hits - hits_{random}} {hits + misses + false\ alarms - hits_{random}} where - hits_{random} = \frac{(hits + misses)(hits + false\ alarms)}{total count} + hits_{random} = frac{(hits + misses)(hits + false\ alarms)}{total count} ETS ranges from -1/3 to 1, where 1 indicates a perfect forecast, 0 indicates no skill beyond random chance, and negative values indicate worse than random chance. From 953de9ec99ada2abd1f1405be3854a8512f02f25 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 10:08:46 +0100 Subject: [PATCH 14/17] add tolerance args --- tests/operators/test_scoreswrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 36f413856..21b2661d6 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -565,7 +565,7 @@ def test_ets_gt_perfect_forecast(make_cube_categorical_testing): ) assert len(result) == 1 - assert np.allclose(result[0].data, 1.0) + assert np.allclose(result[0].data, 1.0, atol=1e-2, rtol=1e-6) def test_ets_gt_mixed_case(make_cube_categorical_testing): From b20a812ba92b896248d73f3b97db64ffbfc73de3 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 10:25:25 +0100 Subject: [PATCH 15/17] tidy up tests --- tests/operators/test_scoreswrappers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/operators/test_scoreswrappers.py b/tests/operators/test_scoreswrappers.py index 21b2661d6..b3673425e 100644 --- a/tests/operators/test_scoreswrappers.py +++ b/tests/operators/test_scoreswrappers.py @@ -603,7 +603,7 @@ def test_ets_gt_mixed_case(make_cube_categorical_testing): # ETS=(1-1)/(1+1+1-1)=0 assert len(result) == 1 - assert np.allclose(result[0].data, 0.0) + assert np.allclose(result[0].data, 0.0, atol=1e-2, rtol=1e-6) def test_ets_gt_complete_miss(make_cube_categorical_testing): @@ -633,7 +633,7 @@ def test_ets_gt_complete_miss(make_cube_categorical_testing): # ETS=(0-1)/(0+2+2-1)=-1/3 assert len(result) == 1 - assert np.allclose(result[0].data, -1.0 / 3.0) + assert np.allclose(result[0].data, -1.0 / 3.0, atol=1e-2, rtol=1e-6) def test_ets_metadata(make_cube_categorical_testing): From a06aa3ad4000b9153935910a6e393996a2bacdd6 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 11:11:12 +0100 Subject: [PATCH 16/17] remove triggers --- src/CSET/cset_workflow/meta/verification/rose-meta.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/CSET/cset_workflow/meta/verification/rose-meta.conf b/src/CSET/cset_workflow/meta/verification/rose-meta.conf index 6cb9d3d66..9dd2dbcc6 100644 --- a/src/CSET/cset_workflow/meta/verification/rose-meta.conf +++ b/src/CSET/cset_workflow/meta/verification/rose-meta.conf @@ -346,7 +346,6 @@ ns=Verification/Scores Categorical description=Compute Scores Probability of Detection. type=python_boolean compulsory=true -trigger=template variables=SCORES_CATEGORICAL_POD_ENTRIES: True sort-key=scoresPOD2 [Verification/Scores Categorical] @@ -355,5 +354,4 @@ ns=Verification/Scores Categorical description=Compute Scores Equitable Threat Score. type=python_boolean compulsory=true -trigger=template variables=SCORES_CATEGORICAL_ETS_ENTRIES: True sort-key=scoresPOD3 From 98423a8bfecb1112a4c030582d946de738306a80 Mon Sep 17 00:00:00 2001 From: James Warner Date: Thu, 20 Aug 2026 16:22:35 +0100 Subject: [PATCH 17/17] update recipes with plot construction info and additional references --- .../verification/surface_categorical_model_obs_ets.yaml | 5 ++++- .../verification/surface_categorical_model_obs_pod.yaml | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml index 4ee74cb92..6d9385664 100644 --- a/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml +++ b/src/CSET/recipes/verification/surface_categorical_model_obs_ets.yaml @@ -2,7 +2,10 @@ category: Scores Categorical title: Timeseries of ETS for "$VARNAME over $SUBAREA_NAME, using threshold $OPERATOR $THRESHOLD" description: | - The Equitable Threat Score (ETS) evaluates the accuracy of forecasts for events that meet a specified threshold, + Extracts and plots the ETS for $VARNAME for each model against observations as a timeseries. + + The Equitable Threat Score (ETS) in [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.categorical.BasicContingencyManager.equitable_threat_score) + evaluates the accuracy of forecasts for events that meet a specified threshold, hile accounting for correct forecasts that could occur purely by chance. Unlike the Probability of Detection (POD), ETS considers hits, misses, and false alarms, providing a more balanced assessment of forecast skill. diff --git a/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml b/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml index 6791f2b7e..ba84f85fe 100644 --- a/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml +++ b/src/CSET/recipes/verification/surface_categorical_model_obs_pod.yaml @@ -2,9 +2,10 @@ category: Scores Categorical title: Timeseries of POD for "$VARNAME over $SUBAREA_NAME, using threshold $OPERATOR $THRESHOLD" description: | - Create a timeseries of the probability of detection. + Extracts and plots the POD for $VARNAME for each model against observations as a timeseries. - The probability of detection calculates the proportion of observed events that meet a threshold that were correctly forecast by the model. + The probability of detection (POD) in [`scores`](https://scores.readthedocs.io/en/stable/api.html#scores.categorical.BasicContingencyManager.probability_of_detection) + calculates the proportion of observed events that meet a threshold that were correctly forecast by the model. For example, if threshold is 290K and op_func is gt (greater than), and at some station a temperature was recorded as 292K and the model produced 295k, that would be a positive hit. It does not take into account how far above/below a threshold a model forecasts.