Detangle generation data

src/ocf_data_sampler/config/__init__.py

@@ -1,5 +1,5 @@
 """Configuration model."""
 
 from ocf_data_sampler.config.load import load_yaml_configuration
-from ocf_data_sampler.config.model import Configuration, InputData
+from ocf_data_sampler.config.model import PVNetDataConfig
 from ocf_data_sampler.config.save import save_yaml_configuration

src/ocf_data_sampler/config/load.py

@@ -3,10 +3,10 @@
 import fsspec
 from pyaml_env import parse_config
 
-from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.config.model import PVNetDataConfig
 
 
-def load_yaml_configuration(filename: str) -> Configuration:
+def load_yaml_configuration(filename: str) -> PVNetDataConfig:
     """Load a yaml file which has a configuration in it.
 
     Args:
@@ -19,4 +19,4 @@ def load_yaml_configuration(filename: str) -> Configuration:
     with fsspec.open(filename, mode="r") as stream:
         configuration = parse_config(data=stream)
 
-    return Configuration(**configuration)
+    return PVNetDataConfig(**configuration)

src/ocf_data_sampler/config/model.py

@@ -1,8 +1,4 @@
-"""Configuration model for the dataset.
-
-Absolute or relative zarr filepath(s).
-Prefix with a protocol like s3:// to read from alternative filesystems.
-"""
+"""Configuration model for the PVNet dataset."""
 
 from collections.abc import Iterator
 from typing import Literal
@@ -20,16 +16,6 @@ class Base(BaseModel):
     model_config = ConfigDict(extra="forbid")
 
 
-class General(Base):
-    """General pydantic model."""
-
-    name: str = Field("example", description="The name of this configuration file")
-    description: str = Field(
-        "example configuration",
-        description="Description of this configuration file",
-    )
-
-
 class TimeWindowMixin(Base):
     """Mixin class, to add interval start, end and resolution minutes."""
 
@@ -72,7 +58,16 @@ def validate_intervals(self) -> "TimeWindowMixin":
         return self
 
 
-class DropoutMixin(Base):
+class FillValueMixin(Base):
+    """Mixin class, to add a value used for filling missing data."""
+
+    dropout_fill_value: float = Field(
+        default=0.0,
+        description="The value used to fill in dropped out data or any missing values."
+    )
+
+
+class DropoutMixin(FillValueMixin):
     """Mixin class, to add dropout minutes."""
 
     dropout_timedeltas_minutes: list[int] = Field(
@@ -87,11 +82,6 @@ class DropoutMixin(Base):
         "floats (probability that dropout of the corresponding timedelta is applied)",
     )
 
-    dropout_fill_value: float = Field(
-        default=0.0,
-        description="The value used to fill in dropped out data or any missing values."
-    )
-
     @field_validator("dropout_timedeltas_minutes")
     def dropout_timedeltas_minutes_negative(cls, v: list[int]) -> list[int]:
         """Validate 'dropout_timedeltas_minutes'."""
@@ -235,7 +225,7 @@ class NWP(TimeWindowMixin, DropoutMixin, SpatialWindowMixin, NormalisationConsta
 
     provider: str = Field(..., description="The provider of the NWP data")
 
-    accum_channels: list[str] = Field([], description="the nwp channels which need to be diffed")
+    accum_channels: list[str] = Field([], description="The NWP channels which need to be diffed")
 
     max_staleness_minutes: int | None = Field(
         None,
@@ -319,7 +309,48 @@ def items(self) -> Iterator[tuple[str, NWP]]:
         return self.root.items()
 
 
-class Generation(TimeWindowMixin, DropoutMixin):
+class GenerationWindow(Base):
+    """Interval bounds for a generation window.
+
+    Unlike `TimeWindowMixin`, no temporal resolution here - it belongs to the shared
+    generation data source (`Generation.time_resolution_minutes`), not to an individual window.
+    """
+
+    interval_start_minutes: int = Field(
+        ...,
+        description="Data interval starts at `t0 + interval_start_minutes`",
+    )
+
+    interval_end_minutes: int = Field(
+        ...,
+        description="Data interval ends at `t0 + interval_end_minutes`",
+    )
+
+    @model_validator(mode="after")
+    def validate_interval_order(self) -> "GenerationWindow":
+        """Validator for time interval fields."""
+        start = self.interval_start_minutes
+        end = self.interval_end_minutes
+        if start > end:
+            raise ValueError(
+                f"interval_start_minutes ({start}) must be <= interval_end_minutes ({end})",
+            )
+        return self
+
+
+class GenerationInputWindow(GenerationWindow, DropoutMixin):
+    """A generation window with dropout configuration.
+
+    Dropout is configurable here since only the input window (not the prediction target)
+    should ever be randomly masked out.
+    """
+
+
+class GenerationTargetWindow(GenerationWindow, FillValueMixin):
+    """Generation target window configuration model."""
+
+
+class Generation(Base):
     """Generation configuration model."""
 
     zarr_path: str = Field(
@@ -328,11 +359,77 @@ class Generation(TimeWindowMixin, DropoutMixin):
         "to read from alternative filesystems.",
     )
 
+    time_resolution_minutes: int = Field(
+        ...,
+        gt=0,
+        description="The temporal resolution of the generation data in minutes",
+    )
+
+    input: GenerationInputWindow | None = None
+    target: GenerationTargetWindow | None = None
+
+    @model_validator(mode="after")
+    def validate_windows(self) -> "Generation":
+        """Validate the input/target windows are set and divisible by the shared resolution."""
+        if self.input is None and self.target is None:
+            raise ValueError(
+                "At least one of `generation.input` or `generation.target` must be configured",
+            )
+
+        for name, window in (("input", self.input), ("target", self.target)):
+            if window is None:
+                continue
+            for bound_name, bound in (
+                ("interval_start_minutes", window.interval_start_minutes),
+                ("interval_end_minutes", window.interval_end_minutes),
+            ):
+                if bound % self.time_resolution_minutes != 0:
+                    raise ValueError(
+                        f"generation.{name}.{bound_name} ({bound}) must be divisible by "
+                        f"generation.time_resolution_minutes ({self.time_resolution_minutes})",
+                    )
+        return self
+
+
+class SamplingGrid(Base):
+    """Configuration for the (location, time) grid that t0 times are sampled from."""
+
+    locations_csv_path: str = Field(
+        ...,
+        description="Absolute or relative CSV filepath to the locations metadata (location IDs "
+        "and their coordinates) - see `ocf_data_sampler.load.locations.open_locations`.",
+    )
+
+    exclude_location_ids: list[int] = Field(
+        default=[],
+        description="Location IDs from the locations metadata to drop from the sampling grid. "
+        "Every ID listed must be present in the locations data.",
+    )
+
+    t0_resolution_minutes: int = Field(
+        ...,
+        gt=0,
+        description="The cadence t0 candidates are enumerated at, needed to compute valid t0 "
+        "times regardless of which other input sources are configured.",
+    )
+
+    @field_validator("exclude_location_ids")
+    def validate_exclude_location_ids_unique(cls, v: list[int]) -> list[int]:
+        """Validate 'exclude_location_ids'."""
+        duplicates = {i for i in v if v.count(i) > 1}
+        if duplicates:
+            raise ValueError(f"exclude_location_ids contains duplicates: {sorted(duplicates)}")
+        return v
+
 
 class SolarPosition(TimeWindowMixin):
     """Solar position configuration model."""
 
 
+class DatetimeEncoding(TimeWindowMixin):
+    """Datetime encoding configuration model."""
+
+
 _embedding_type = list[tuple[str, Literal["cyclic", "linear"]]]
 class T0Embedding(Base):
     """Configuration for the t0 time embedding."""
@@ -374,18 +471,13 @@ def validate_embeddings(cls, embeddings: _embedding_type) -> _embedding_type:
         return embeddings
 
 
-class InputData(Base):
-    """Input data model."""
+class PVNetDataConfig(Base):
+    """Configuration model for the PVNet dataset."""
 
+    sampling_grid: SamplingGrid
     satellite: Satellite | None = None
     nwp: MultiNWP | None = None
     generation: Generation | None = None
     solar_position: SolarPosition | None = None
+    datetime_encoding: DatetimeEncoding | None = None
     t0_embedding: T0Embedding | None = None
-
-
-class Configuration(Base):
-    """Configuration model for the dataset."""
-
-    general: General = General()
-    input_data: InputData = InputData()

src/ocf_data_sampler/config/save.py

@@ -10,14 +10,14 @@
 import fsspec
 import yaml
 
-from ocf_data_sampler.config.model import Configuration
+from ocf_data_sampler.config.model import PVNetDataConfig
 
 
-def save_yaml_configuration(configuration: Configuration, filename: str) -> None:
+def save_yaml_configuration(configuration: PVNetDataConfig, filename: str) -> None:
     """Save a configuration object to a YAML file.
 
     Args:
-        configuration: Configuration object containing the settings to save
+        configuration: PVNetDataConfig object containing the settings to save
         filename: Destination path for the YAML file. Can be a local path or
                  cloud storage URL (e.g., 'gs://', 's3://'). For local paths,
                  absolute paths are recommended.