From 19b9f0185eea1b49b4a618d4b245f6aa63434ce3 Mon Sep 17 00:00:00 2001 From: lm2612 Date: Wed, 29 Jul 2026 15:55:14 +0000 Subject: [PATCH 1/6] store quantile losses to plot q quantile quantile plot in wandb --- pvnet/training/lightning_module.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pvnet/training/lightning_module.py b/pvnet/training/lightning_module.py index 18678b78..0d5430e6 100644 --- a/pvnet/training/lightning_module.py +++ b/pvnet/training/lightning_module.py @@ -129,11 +129,15 @@ def _calculate_val_losses( if self.model.use_quantile_regression: metric_name = "val_fraction_below/fraction_below_{:.2f}_quantile" # Add fraction below each quantile for calibration + val_quantiles = np.zeros(len(self.model.output_quantiles)) for i, quantile in enumerate(self.model.output_quantiles): below_quant = y <= y_hat[..., i] # Mask values small values, which are dominated by night mask = y >= 0.01 - losses[metric_name.format(quantile)] = below_quant[mask].float().mean() + below_quant_masked_mean = below_quant[mask].float().mean() + losses[metric_name.format(quantile)] = below_quant_masked_mean + val_quantiles[i] = below_quant_masked_mean + self._val_quantiles.append(val_quantiles) return losses @@ -185,6 +189,8 @@ def on_validation_epoch_start(self): # Set up stores which we will fill during validation self.all_val_results: list[xr.Dataset] = [] self._val_horizon_maes: list[np.array] = [] + if self.model.use_quantile_regression: + self._val_quantiles: list[np.array] = [] if self.current_epoch == 0: self._val_persistence_horizon_maes: list[np.array] = [] @@ -355,6 +361,24 @@ def on_validation_epoch_end(self) -> None: step=self.trainer.global_step, ) + # Create a quantile-quantile plot + if self.model.use_quantile_regression: + val_quantiles = np.mean(self._val_quantiles, axis=0) + self._val_quantiles = [] + + qq_plot = wandb_line_plot( + x=self.model.output_quantiles, + y=val_quantiles, + xlabel="True quantiles", + ylabel="Predicted quantiles", + title="Quantile-quantile plot", + ) + + wandb.log( + {"quantile_quantile": qq_plot}, + step=self.trainer.global_step, + ) + # Create persistence horizon accuracy curve but only on first epoch if self.current_epoch == 0: persist_horizon_mae_plot = wandb_line_plot( From f37b14010408abeac43bb075395d89cebbdc572d Mon Sep 17 00:00:00 2001 From: lm2612 Date: Wed, 29 Jul 2026 16:38:46 +0000 Subject: [PATCH 2/6] chnaged test to quantile --- tests/training/test_train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/training/test_train.py b/tests/training/test_train.py index 1760604d..c3f0e47e 100644 --- a/tests/training/test_train.py +++ b/tests/training/test_train.py @@ -20,7 +20,7 @@ def trainer_cfg_cpu() -> dict: """Tiny CPU-only Trainer config.""" return { "_target_": "lightning.pytorch.Trainer", - "max_epochs": 1, + "max_epochs": 2, "limit_train_batches": 1, "limit_val_batches": 1, "accelerator": "cpu", @@ -71,6 +71,7 @@ def build_lit_late_fusion_cfg( "_target_": "pvnet.training.lightning_module.PVNetLightningModule", "model": { "_target_": "pvnet.models.LateFusionModel", + "output_quantiles": [0.1, 0.5, 0.9], "sat_encoder": None, "nwp_encoders_dict": None, "add_image_embedding_channel": False, From 9d7b258a4017b612f35bd2fa72110afd1cf84d82 Mon Sep 17 00:00:00 2001 From: lm2612 Date: Thu, 6 Aug 2026 13:25:21 +0000 Subject: [PATCH 3/6] trying custom spec to be able to add custom y=x line --- pvnet/training/lightning_module.py | 4 +-- pvnet/training/plots.py | 43 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pvnet/training/lightning_module.py b/pvnet/training/lightning_module.py index 0d5430e6..230d6710 100644 --- a/pvnet/training/lightning_module.py +++ b/pvnet/training/lightning_module.py @@ -14,7 +14,7 @@ from pvnet.datamodule import collate_fn from pvnet.models.base_model import BaseModel from pvnet.optimizers import AbstractOptimizer -from pvnet.training.plots import plot_sample_forecasts, wandb_line_plot +from pvnet.training.plots import plot_sample_forecasts, wandb_line_plot, wandb_line_plot_custom from pvnet.utils import validate_batch_against_config @@ -366,7 +366,7 @@ def on_validation_epoch_end(self) -> None: val_quantiles = np.mean(self._val_quantiles, axis=0) self._val_quantiles = [] - qq_plot = wandb_line_plot( + qq_plot = wandb_line_plot_custom( x=self.model.output_quantiles, y=val_quantiles, xlabel="True quantiles", diff --git a/pvnet/training/plots.py b/pvnet/training/plots.py index 21121a37..3835d468 100644 --- a/pvnet/training/plots.py +++ b/pvnet/training/plots.py @@ -21,6 +21,49 @@ def wandb_line_plot( table = wandb.Table(data=data, columns=[xlabel, ylabel]) return wandb.plot.line(table, xlabel, ylabel, title=title) +def wandb_line_plot_custom( + x: Sequence[float], + y: Sequence[float], + xlabel: str, + ylabel: str, + title: str | None = None + ) -> wandb.plot.CustomChart: + """Make a custom wandb line plot""" + data = [[xi, yi] for (xi, yi) in zip(x, y)] + table = wandb.Table(data=data, columns=[xlabel, ylabel]) + + custom_spec = { + "$schema": "https://vega.github.io/schema/vega-lite/v4.json", + "data": {"name": "wandb"}, + "mark": { + "type": "line", + "strokeWidth": 2 + }, + "encoding": { + "x": {"field": "x_val", "type": "quantitative", "title": xlabel}, + "y": {"field": "y_val", "type": "quantitative", "title": ylabel}, + "color": { + "field": "Series", + "type": "nominal", + }, + "strokeDash": { + "field": "Series", + "type": "nominal", + "condition": {"test": "datum.Series === 'x=y'", "value": [6, 6]}, + "value": [0, 0] + } + }, + "title": title or "" + } + + return wandb.plot_table( + vega_spec_name="custom_line", + data_table=table, + string_fields={"title": title or ""}, + custom_plan=custom_spec + ) + + def plot_sample_forecasts( batch: TensorBatch, From a19ec2796f8739c9308ce98503ec003eb5adb697 Mon Sep 17 00:00:00 2001 From: lm2612 Date: Fri, 7 Aug 2026 15:41:02 +0000 Subject: [PATCH 4/6] custom line plot for y=x line --- pvnet/training/plots.py | 49 +++++++++-------------------------------- 1 file changed, 11 insertions(+), 38 deletions(-) diff --git a/pvnet/training/plots.py b/pvnet/training/plots.py index 3835d468..e43d7a77 100644 --- a/pvnet/training/plots.py +++ b/pvnet/training/plots.py @@ -23,47 +23,20 @@ def wandb_line_plot( def wandb_line_plot_custom( x: Sequence[float], - y: Sequence[float], - xlabel: str, - ylabel: str, - title: str | None = None + y: Sequence[float], + xlabel: str, + ylabel: str, + title: str | None = None ) -> wandb.plot.CustomChart: - """Make a custom wandb line plot""" - data = [[xi, yi] for (xi, yi) in zip(x, y)] - table = wandb.Table(data=data, columns=[xlabel, ylabel]) - - custom_spec = { - "$schema": "https://vega.github.io/schema/vega-lite/v4.json", - "data": {"name": "wandb"}, - "mark": { - "type": "line", - "strokeWidth": 2 - }, - "encoding": { - "x": {"field": "x_val", "type": "quantitative", "title": xlabel}, - "y": {"field": "y_val", "type": "quantitative", "title": ylabel}, - "color": { - "field": "Series", - "type": "nominal", - }, - "strokeDash": { - "field": "Series", - "type": "nominal", - "condition": {"test": "datum.Series === 'x=y'", "value": [6, 6]}, - "value": [0, 0] - } - }, - "title": title or "" - } - - return wandb.plot_table( - vega_spec_name="custom_line", - data_table=table, - string_fields={"title": title or ""}, - custom_plan=custom_spec + """Make a wandb plot with data and an x=y reference line.""" + return wandb.plot.line_series( + xs=[list(x), [0., 1.]], + ys=[list(y), [0., 1.]], + keys=["Data", "x=y"], + title=title, + xname=xlabel ) - def plot_sample_forecasts( batch: TensorBatch, From 208d516a0cbc342dbac67e970fd052deef1e1c3e Mon Sep 17 00:00:00 2001 From: lm2612 Date: Fri, 7 Aug 2026 16:23:56 +0000 Subject: [PATCH 5/6] merged into one custom line plot --- pvnet/training/plots.py | 42 ++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/pvnet/training/plots.py b/pvnet/training/plots.py index e43d7a77..0e62c0b1 100644 --- a/pvnet/training/plots.py +++ b/pvnet/training/plots.py @@ -14,29 +14,41 @@ def wandb_line_plot( y: Sequence[float], xlabel: str, ylabel: str, - title: str | None = None + title: str | None = None, + add_identity_line: bool = False, ) -> wandb.plot.CustomChart: """Make a wandb line plot""" - data = [[xi, yi] for (xi, yi) in zip(x, y)] - table = wandb.Table(data=data, columns=[xlabel, ylabel]) - return wandb.plot.line(table, xlabel, ylabel, title=title) + # Main series data + data = [[xi, yi, "Data"] for xi, yi in zip(x, y)] + + # Add identity line endpoints if requested + if add_identity_line: + min_val, max_val = min(x), max(x) + data.append([min_val, min_val, "x=y"]) + data.append([max_val, max_val, "x=y"]) + + table = wandb.Table(data=data, columns=[xlabel, ylabel, "Series"]) + + # stroke=None creates a clean single line; stroke="Series" creates multi-line legend + stroke_col = "Series" if add_identity_line else None + + return wandb.plot.line( + table=table, + x=xlabel, + y=ylabel, + stroke=stroke_col, + title=title + ) def wandb_line_plot_custom( x: Sequence[float], y: Sequence[float], xlabel: str, ylabel: str, - title: str | None = None - ) -> wandb.plot.CustomChart: - """Make a wandb plot with data and an x=y reference line.""" - return wandb.plot.line_series( - xs=[list(x), [0., 1.]], - ys=[list(y), [0., 1.]], - keys=["Data", "x=y"], - title=title, - xname=xlabel - ) - + title: str | None = None): + return wandb_line_plot(x=x, y=y, xlabel=xlabel, ylabel=ylabel, + title=title, add_identity_line=True) + def plot_sample_forecasts( batch: TensorBatch, From 95d5490afb37439fd45fdb9ef697d3c223d803a6 Mon Sep 17 00:00:00 2001 From: lm2612 Date: Mon, 10 Aug 2026 14:53:54 +0000 Subject: [PATCH 6/6] moved add_identity_line into lightning_module rather than as separate custom function --- pvnet/training/lightning_module.py | 3 ++- pvnet/training/plots.py | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/pvnet/training/lightning_module.py b/pvnet/training/lightning_module.py index 230d6710..484ba61b 100644 --- a/pvnet/training/lightning_module.py +++ b/pvnet/training/lightning_module.py @@ -366,12 +366,13 @@ def on_validation_epoch_end(self) -> None: val_quantiles = np.mean(self._val_quantiles, axis=0) self._val_quantiles = [] - qq_plot = wandb_line_plot_custom( + qq_plot = wandb_line_plot( x=self.model.output_quantiles, y=val_quantiles, xlabel="True quantiles", ylabel="Predicted quantiles", title="Quantile-quantile plot", + add_identity_line=True, ) wandb.log( diff --git a/pvnet/training/plots.py b/pvnet/training/plots.py index 0e62c0b1..3e37f90a 100644 --- a/pvnet/training/plots.py +++ b/pvnet/training/plots.py @@ -40,16 +40,6 @@ def wandb_line_plot( title=title ) -def wandb_line_plot_custom( - x: Sequence[float], - y: Sequence[float], - xlabel: str, - ylabel: str, - title: str | None = None): - return wandb_line_plot(x=x, y=y, xlabel=xlabel, ylabel=ylabel, - title=title, add_identity_line=True) - - def plot_sample_forecasts( batch: TensorBatch, y_hat: torch.Tensor,