From 2eb0e7d3f26d61a5458e2ee8b18676e9f9786aca Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:18:29 -0300 Subject: [PATCH 01/10] feat(data): extend manifest schema to v2 Schema v2 keeps v1 JSONL readable and makes categorical emotion optional, so a corpus can contribute dimensional or auxiliary targets without being forced into an eight-class label it does not have. Rows can now carry normalized VAD, social attitude, binary affect, language, transcript, per-target source and confidence, speaker/session identity, native split, segment bounds, normalized-PCM SHA-256, dataset revision, license, and source provenance. Callers that require a categorical label ask for it through require_label() rather than assuming it is present. --- ser/_internal/data/__init__.py | 6 +- ser/_internal/data/manifest.py | 294 ++++++++++++++---- tests/suites/unit/data/test_manifest_jsonl.py | 114 ++++++- 3 files changed, 355 insertions(+), 59 deletions(-) diff --git a/ser/_internal/data/__init__.py b/ser/_internal/data/__init__.py index c35809e..896fe93 100644 --- a/ser/_internal/data/__init__.py +++ b/ser/_internal/data/__init__.py @@ -8,14 +8,18 @@ load_utterances, ) from ser._internal.data.embedding_cache import EmbeddingCache, EmbeddingCacheEntry -from ser._internal.data.manifest import Utterance +from ser._internal.data.manifest import TargetAnnotation, Utterance, VadTarget +from ser._internal.data.recipe import DatasetRecipe __all__ = [ "DataSplit", "EmbeddingCache", "EmbeddingCacheEntry", + "DatasetRecipe", "LabeledAudioSample", + "TargetAnnotation", "Utterance", + "VadTarget", "load_data", "load_labeled_audio_paths", "load_utterances", diff --git a/ser/_internal/data/manifest.py b/ser/_internal/data/manifest.py index 31841fb..b0d132c 100644 --- a/ser/_internal/data/manifest.py +++ b/ser/_internal/data/manifest.py @@ -1,7 +1,9 @@ -"""Manifest schema for multi-corpus SER training.""" +"""Versioned manifest schema for cross-domain SER training.""" from __future__ import annotations +import math +import re from collections.abc import Mapping from dataclasses import dataclass from pathlib import Path @@ -9,9 +11,14 @@ from ser._internal.data.ontology import LabelOntology, ensure_label_allowed, normalize_label -MANIFEST_SCHEMA_VERSION = 1 +MANIFEST_SCHEMA_VERSION = 2 +SUPPORTED_MANIFEST_SCHEMA_VERSIONS = frozenset({1, MANIFEST_SCHEMA_VERSION}) +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") type SplitName = Literal["train", "dev", "test"] +type AnnotationTarget = Literal[ + "emotion", "vad", "social_attitude", "binary_affect", "language", "text" +] def _read_text_field(record: Mapping[str, object], field: str) -> str | None: @@ -23,11 +30,20 @@ def _read_text_field(record: Mapping[str, object], field: str) -> str | None: def _read_float_field(record: Mapping[str, object], field: str) -> float | None: raw = record.get(field) - if isinstance(raw, int | float): + if isinstance(raw, int | float) and not isinstance(raw, bool): return float(raw) return None +def _read_optional_float_field(record: Mapping[str, object], field: str) -> float | None: + if field not in record or record.get(field) is None: + return None + value = _read_float_field(record, field) + if value is None: + raise ValueError(f"Manifest {field!r} must be numeric when provided.") + return value + + def _resolve_audio_path(path_text: str, base_dir: Path) -> Path: candidate = Path(path_text).expanduser() if candidate.is_absolute(): @@ -38,56 +54,198 @@ def _resolve_audio_path(path_text: str, base_dir: Path) -> Path: def _maybe_relative(path: Path, base_dir: Path) -> Path: try: return path.relative_to(base_dir) - except Exception: + except ValueError: return path +@dataclass(frozen=True) +class VadTarget: + """Normalized valence, arousal, and dominance target in ``[-1, 1]``.""" + + valence: float + arousal: float + dominance: float + + def validate(self) -> None: + """Validates finite normalized VAD coordinates.""" + for name, value in ( + ("valence", self.valence), + ("arousal", self.arousal), + ("dominance", self.dominance), + ): + if not math.isfinite(value) or not -1.0 <= value <= 1.0: + raise ValueError(f"VAD {name} must be finite and within [-1, 1].") + + @staticmethod + def from_record(raw: object) -> VadTarget | None: + """Parses an optional VAD object from a manifest record.""" + if raw is None: + return None + if not isinstance(raw, dict): + raise ValueError("Manifest 'vad' target must be an object.") + values: list[float] = [] + for field in ("valence", "arousal", "dominance"): + value = raw.get(field) + if not isinstance(value, int | float) or isinstance(value, bool): + raise ValueError(f"Manifest 'vad.{field}' must be numeric.") + values.append(float(value)) + target = VadTarget(*values) + target.validate() + return target + + def to_record(self) -> dict[str, float]: + """Serializes VAD coordinates.""" + return { + "valence": self.valence, + "arousal": self.arousal, + "dominance": self.dominance, + } + + +@dataclass(frozen=True) +class TargetAnnotation: + """Source and confidence metadata for one available target.""" + + target: AnnotationTarget + source: str + confidence: float | None = None + + def validate(self) -> None: + """Validates annotation source and confidence.""" + if self.target not in { + "emotion", + "vad", + "social_attitude", + "binary_affect", + "language", + "text", + }: + raise ValueError(f"Unsupported annotation target {self.target!r}.") + if not self.source.strip(): + raise ValueError("Annotation source must be non-empty.") + if self.confidence is not None and ( + not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0 + ): + raise ValueError("Annotation confidence must be finite and within [0, 1].") + + @staticmethod + def from_record(raw: object) -> TargetAnnotation: + """Parses annotation metadata from a JSON object.""" + if not isinstance(raw, dict): + raise ValueError("Manifest annotations must contain objects.") + target = _read_text_field(raw, "target") + source = _read_text_field(raw, "source") + confidence = _read_float_field(raw, "confidence") + if "confidence" in raw and raw.get("confidence") is not None and confidence is None: + raise ValueError("Manifest annotation confidence must be numeric when provided.") + if target is None or source is None: + raise ValueError("Manifest annotations require target and source fields.") + annotation = TargetAnnotation(cast(AnnotationTarget, target), source, confidence) + annotation.validate() + return annotation + + def to_record(self) -> dict[str, object]: + """Serializes annotation metadata.""" + record: dict[str, object] = {"target": self.target, "source": self.source} + if self.confidence is not None: + record["confidence"] = self.confidence + return record + + @dataclass(frozen=True) class Utterance: - """One supervised SER training record.""" + """One audio segment and any targets available for training.""" schema_version: int sample_id: str corpus: str audio_path: Path - label: str + label: str | None raw_label: str | None = None + vad: VadTarget | None = None + social_attitude: str | None = None + binary_affect: str | None = None + transcript: str | None = None + annotations: tuple[TargetAnnotation, ...] = () speaker_id: str | None = None + session_id: str | None = None language: str | None = None split: SplitName | None = None + native_split: SplitName | None = None start_seconds: float | None = None duration_seconds: float | None = None + normalized_audio_sha256: str | None = None + dataset_revision: str | None = None dataset_policy_id: str | None = None dataset_license_id: str | None = None source_url: str | None = None + def require_label(self) -> str: + """Returns the primary label or raises at a supervised-only boundary.""" + if self.label is None: + raise ValueError(f"Utterance {self.sample_id!r} has no primary emotion target.") + return self.label + def validate(self, *, ontology: LabelOntology) -> None: - """Validates record fields for stability across adapters.""" - if self.schema_version != MANIFEST_SCHEMA_VERSION: + """Validates record fields and target boundaries.""" + if self.schema_version not in SUPPORTED_MANIFEST_SCHEMA_VERSIONS: raise ValueError( f"Unsupported manifest schema version {self.schema_version!r}; " - f"expected {MANIFEST_SCHEMA_VERSION}." + f"supported versions are {sorted(SUPPORTED_MANIFEST_SCHEMA_VERSIONS)}." ) if not self.sample_id.strip(): raise ValueError("Utterance.sample_id must be non-empty.") if not self.corpus.strip(): raise ValueError("Utterance.corpus must be non-empty.") - if not isinstance(self.audio_path, Path): - raise ValueError("Utterance.audio_path must be a Path.") - if not str(self.audio_path).strip(): - raise ValueError("Utterance.audio_path must be non-empty.") - ensure_label_allowed(label=self.label, ontology=ontology) - if self.speaker_id is not None: - expected_prefix = f"{self.corpus}:" - if not self.speaker_id.startswith(expected_prefix): + if not isinstance(self.audio_path, Path) or not str(self.audio_path).strip(): + raise ValueError("Utterance.audio_path must be a non-empty Path.") + if self.label is not None: + ensure_label_allowed(label=self.label, ontology=ontology) + if self.schema_version == 1 and self.label is None: + raise ValueError("Manifest schema v1 requires a categorical label.") + if self.schema_version == 2 and not any( + ( + self.label, + self.vad, + self.social_attitude, + self.binary_affect, + self.language, + self.transcript, + ) + ): + raise ValueError("Manifest schema v2 requires at least one training target.") + expected_prefix = f"{self.corpus}:" + for field_name, identity in ( + ("speaker_id", self.speaker_id), + ("session_id", self.session_id), + ): + if identity is not None and not identity.startswith(expected_prefix): raise ValueError( - "speaker_id must be corpus-scoped to avoid collisions: " - f"expected prefix {expected_prefix!r} in {self.speaker_id!r}." + f"{field_name} must be corpus-scoped to avoid collisions: " + f"expected prefix {expected_prefix!r} in {identity!r}." ) - if self.start_seconds is not None and self.start_seconds < 0.0: - raise ValueError("start_seconds must be non-negative.") - if self.duration_seconds is not None and self.duration_seconds <= 0.0: - raise ValueError("duration_seconds must be positive when provided.") + if self.start_seconds is not None and ( + not math.isfinite(self.start_seconds) or self.start_seconds < 0.0 + ): + raise ValueError("start_seconds must be finite and non-negative.") + if self.duration_seconds is not None and ( + not math.isfinite(self.duration_seconds) or self.duration_seconds <= 0.0 + ): + raise ValueError("duration_seconds must be finite and positive when provided.") + if self.normalized_audio_sha256 is not None and not _SHA256_PATTERN.fullmatch( + self.normalized_audio_sha256 + ): + raise ValueError("normalized_audio_sha256 must be 64 lowercase hexadecimal characters.") + if self.dataset_revision is not None and not self.dataset_revision.strip(): + raise ValueError("dataset_revision must be non-empty when provided.") + if self.vad is not None: + self.vad.validate() + seen_targets: set[AnnotationTarget] = set() + for annotation in self.annotations: + annotation.validate() + if annotation.target in seen_targets: + raise ValueError(f"Duplicate annotation metadata for {annotation.target!r}.") + seen_targets.add(annotation.target) @staticmethod def from_record( @@ -96,44 +254,59 @@ def from_record( base_dir: Path, ontology: LabelOntology, ) -> Utterance: - """Builds an Utterance from a parsed manifest record.""" - schema_version_raw = record.get("schema_version", MANIFEST_SCHEMA_VERSION) - schema_version = ( - int(schema_version_raw) - if isinstance(schema_version_raw, int) - else MANIFEST_SCHEMA_VERSION - ) + """Builds an utterance from a v1 or v2 parsed manifest record.""" + schema_version_raw = record.get("schema_version", 1) + if not isinstance(schema_version_raw, int) or isinstance(schema_version_raw, bool): + raise ValueError("Manifest schema_version must be an integer.") + if schema_version_raw not in SUPPORTED_MANIFEST_SCHEMA_VERSIONS: + raise ValueError( + f"Unsupported manifest schema version {schema_version_raw!r}; " + f"supported versions are {sorted(SUPPORTED_MANIFEST_SCHEMA_VERSIONS)}." + ) sample_id = _read_text_field(record, "sample_id") corpus = _read_text_field(record, "corpus") audio_path_text = _read_text_field(record, "audio_path") or _read_text_field(record, "path") - label_text = _read_text_field(record, "label") - if sample_id is None or corpus is None or audio_path_text is None or label_text is None: + if sample_id is None or corpus is None or audio_path_text is None: raise ValueError( - "Manifest record must include sample_id, corpus, audio_path, and label fields." + "Manifest record must include sample_id, corpus, and audio_path fields." ) - label = normalize_label(label_text) - speaker_id = _read_text_field(record, "speaker_id") - language = _read_text_field(record, "language") + label_text = _read_text_field(record, "label") + if schema_version_raw == 1 and label_text is None: + raise ValueError("Manifest schema v1 requires a categorical label.") + label = normalize_label(label_text) if label_text is not None else None split_raw = _read_text_field(record, "split") - split: SplitName | None = ( - cast(SplitName, split_raw) if split_raw in {"train", "dev", "test"} else None - ) - start_seconds = _read_float_field(record, "start_seconds") - duration_seconds = _read_float_field(record, "duration_seconds") + native_split_raw = _read_text_field(record, "native_split") + annotations_raw = record.get("annotations", []) + if not isinstance(annotations_raw, list): + raise ValueError("Manifest 'annotations' must be a list.") + annotations = tuple(TargetAnnotation.from_record(raw) for raw in annotations_raw) utterance = Utterance( - schema_version=schema_version, + schema_version=MANIFEST_SCHEMA_VERSION, sample_id=sample_id, corpus=corpus, audio_path=_resolve_audio_path(audio_path_text, base_dir), label=label, raw_label=_read_text_field(record, "raw_label"), - speaker_id=speaker_id, - language=language, - split=split, - start_seconds=start_seconds, - duration_seconds=duration_seconds, + vad=VadTarget.from_record(record.get("vad")), + social_attitude=_read_text_field(record, "social_attitude"), + binary_affect=_read_text_field(record, "binary_affect"), + transcript=_read_text_field(record, "transcript"), + annotations=annotations, + speaker_id=_read_text_field(record, "speaker_id"), + session_id=_read_text_field(record, "session_id"), + language=_read_text_field(record, "language"), + split=cast(SplitName, split_raw) if split_raw in {"train", "dev", "test"} else None, + native_split=( + cast(SplitName, native_split_raw) + if native_split_raw in {"train", "dev", "test"} + else None + ), + start_seconds=_read_optional_float_field(record, "start_seconds"), + duration_seconds=_read_optional_float_field(record, "duration_seconds"), + normalized_audio_sha256=_read_text_field(record, "normalized_audio_sha256"), + dataset_revision=_read_text_field(record, "dataset_revision"), dataset_policy_id=_read_text_field(record, "dataset_policy_id"), dataset_license_id=_read_text_field(record, "dataset_license_id"), source_url=_read_text_field(record, "source_url"), @@ -142,31 +315,38 @@ def from_record( return utterance def to_record(self, *, base_dir: Path | None = None) -> dict[str, object]: - """Serializes record for JSONL persistence.""" - path = ( - str(_maybe_relative(self.audio_path, base_dir)) - if base_dir is not None - else str(self.audio_path) - ) + """Serializes a v1 or v2 record for JSONL persistence.""" + path = str(_maybe_relative(self.audio_path, base_dir)) if base_dir else str(self.audio_path) record: dict[str, object] = { "schema_version": self.schema_version, "sample_id": self.sample_id, "corpus": self.corpus, "audio_path": path, - "label": self.label, } optional_fields: dict[str, object | None] = { + "label": self.label, "raw_label": self.raw_label, + "vad": self.vad.to_record() if self.vad is not None else None, + "social_attitude": self.social_attitude, + "binary_affect": self.binary_affect, + "transcript": self.transcript, + "annotations": ( + [annotation.to_record() for annotation in self.annotations] + if self.annotations + else None + ), "speaker_id": self.speaker_id, + "session_id": self.session_id, "language": self.language, "split": self.split, + "native_split": self.native_split, "start_seconds": self.start_seconds, "duration_seconds": self.duration_seconds, + "normalized_audio_sha256": self.normalized_audio_sha256, + "dataset_revision": self.dataset_revision, "dataset_policy_id": self.dataset_policy_id, "dataset_license_id": self.dataset_license_id, "source_url": self.source_url, } - for key, value in optional_fields.items(): - if value is not None: - record[key] = value + record.update((key, value) for key, value in optional_fields.items() if value is not None) return record diff --git a/tests/suites/unit/data/test_manifest_jsonl.py b/tests/suites/unit/data/test_manifest_jsonl.py index 2a6776d..60e6e78 100644 --- a/tests/suites/unit/data/test_manifest_jsonl.py +++ b/tests/suites/unit/data/test_manifest_jsonl.py @@ -6,7 +6,12 @@ import pytest -from ser._internal.data.manifest import MANIFEST_SCHEMA_VERSION, Utterance +from ser._internal.data.manifest import ( + MANIFEST_SCHEMA_VERSION, + TargetAnnotation, + Utterance, + VadTarget, +) from ser._internal.data.manifest_jsonl import load_manifest_jsonl, write_manifest_jsonl from ser._internal.data.ontology import LabelOntology, remap_label @@ -69,6 +74,113 @@ def test_manifest_loader_round_trip(tmp_path: Path) -> None: assert loaded[1].duration_seconds == pytest.approx(2.0) +def test_manifest_loader_preserves_v1_compatibility(tmp_path: Path) -> None: + """Explicit v1 records remain loadable after schema v2 is introduced.""" + path = tmp_path / "legacy.jsonl" + path.write_text( + '{"schema_version":1,"sample_id":"legacy:1","corpus":"legacy",' + '"audio_path":"a.wav","label":"happy"}\n', + encoding="utf-8", + ) + + loaded = load_manifest_jsonl(path, ontology=_ontology()) + + assert loaded[0].schema_version == MANIFEST_SCHEMA_VERSION + assert loaded[0].label == "happy" + + +@pytest.mark.parametrize( + ("payload", "error"), + [ + ( + '{"schema_version":999,"sample_id":"x:1","corpus":"x",' + '"audio_path":"a.wav","label":"happy"}\n', + "Unsupported manifest schema version", + ), + ( + '{"schema_version":1,"sample_id":"x:1","corpus":"x",' '"audio_path":"a.wav"}\n', + "schema v1 requires a categorical label", + ), + ], +) +def test_manifest_loader_validates_source_schema_before_v2_upgrade( + tmp_path: Path, + payload: str, + error: str, +) -> None: + """Compatibility upgrades never reinterpret unsupported or malformed source records.""" + path = tmp_path / "source.jsonl" + path.write_text(payload, encoding="utf-8") + + with pytest.raises(ValueError, match=error): + load_manifest_jsonl(path, ontology=_ontology()) + + +def test_manifest_v2_round_trip_preserves_auxiliary_targets(tmp_path: Path) -> None: + """V2 records may omit emotion while preserving typed auxiliary annotations.""" + path = tmp_path / "auxiliary.jsonl" + utterance = Utterance( + schema_version=MANIFEST_SCHEMA_VERSION, + sample_id="escorpus:1", + corpus="escorpus", + audio_path=tmp_path / "a.wav", + label=None, + vad=VadTarget(valence=-0.2, arousal=0.4, dominance=0.1), + transcript="hola", + annotations=(TargetAnnotation("vad", "annotator_consensus", 0.85),), + session_id="escorpus:session-1", + native_split="dev", + normalized_audio_sha256="a" * 64, + dataset_revision="2026-07", + ) + + write_manifest_jsonl(path, [utterance], base_dir=tmp_path) + loaded = load_manifest_jsonl(path, ontology=_ontology(), base_dir=tmp_path) + + assert loaded == [utterance] + + +@pytest.mark.parametrize( + "record,error", + [ + ( + { + "schema_version": 2, + "sample_id": "x:1", + "corpus": "x", + "audio_path": "a.wav", + "vad": {"valence": 2.0, "arousal": 0.0, "dominance": 0.0}, + }, + r"within \[-1, 1\]", + ), + ( + { + "schema_version": 2, + "sample_id": "x:1", + "corpus": "x", + "audio_path": "a.wav", + "language": "en", + "normalized_audio_sha256": "ABC", + }, + "64 lowercase hexadecimal", + ), + ], +) +def test_manifest_v2_rejects_invalid_target_boundaries( + tmp_path: Path, + record: dict[str, object], + error: str, +) -> None: + """External target and digest fields fail closed at manifest load time.""" + import json + + path = tmp_path / "invalid.jsonl" + path.write_text(json.dumps(record) + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match=error): + load_manifest_jsonl(path, ontology=_ontology()) + + def test_remap_label_honors_unknown_policy() -> None: """Unknown label policy should gate adapter remapping behavior.""" ontology_drop = LabelOntology( From aa8dad1171708bf169f873a8c343fef389517fc4 Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:18:29 -0300 Subject: [PATCH 02/10] feat(data): add versioned dataset recipes and auditing Concatenating every configured manifest silently mixes incompatible label ontologies and leaks speakers across splits. A recipe instead declares, per corpus, which labels are exact enough for the public eight-class head, which are only approximate, and which auxiliary tasks a corpus can serve. Approximate labels route to raw_emotion and never reach the primary head, so calm is never inferred from neutral or sleepy. The audit deduplicates by normalized-PCM hash, unions speaker and session ids into identity components, assigns deterministic grouped splits, and refuses to let a group or a content hash appear in more than one supervised split. Rows with no defensible group become ssl_only and cannot enter validation or test. Recipe, manifest, and split-ledger SHA-256 digests make a benchmark reproducible; strict mode rejects missing revisions or hashes, duplicate content, leakage, and a primary task with fewer than two populated classes. --- docs/dataset-recipes.md | 44 +++ ser/_internal/data/dataset_audit.py | 313 ++++++++++++++++++ ser/_internal/data/recipe.py | 313 ++++++++++++++++++ .../unit/data/test_dataset_recipe_audit.py | 157 +++++++++ 4 files changed, 827 insertions(+) create mode 100644 docs/dataset-recipes.md create mode 100644 ser/_internal/data/dataset_audit.py create mode 100644 ser/_internal/data/recipe.py create mode 100644 tests/suites/unit/data/test_dataset_recipe_audit.py diff --git a/docs/dataset-recipes.md b/docs/dataset-recipes.md new file mode 100644 index 0000000..858b97d --- /dev/null +++ b/docs/dataset-recipes.md @@ -0,0 +1,44 @@ +# Versioned cross-domain dataset recipes + +Training can opt into an explicit recipe instead of concatenating every configured manifest: + +```bash +ser --train --dataset-recipe research-v1 --strict-dataset-audit +``` + +The equivalent configuration is `SER_DATASET_RECIPE=research-v1`. Strict auditing defaults to +enabled whenever a recipe is configured; use `SER_STRICT_DATASET_AUDIT=false` only for exploratory +data repair, never benchmark generation. + +## Manifest schema v2 + +Schema v2 keeps v1 JSONL readable and makes categorical emotion optional. Records can carry +normalized VAD, social attitude, binary affect, language, transcript, per-target source/confidence, +speaker/session identity, native split, segment bounds, normalized-PCM SHA-256, dataset revision, +license, policy, and source provenance. + +Strict recipe training requires: + +- a dataset revision and normalized-PCM SHA-256 on every row; +- unique sample IDs and content hashes across all corpora; +- at least two populated exact primary-emotion classes; +- zero speaker/session component or content overlap across train/dev/test; and +- exhaustive accepted/remapped/weak/dropped/missing/quarantined accounting. + +Rows without a defensible speaker/session group are assigned `ssl_only`; they cannot enter +supervised validation or test. Approximate mappings are routed to `raw_emotion`, never the public +eight-class primary head. The `calm` output remains part of the runtime contract and is not inferred +from neutral or sleepy labels. + +## Reproducibility artifacts + +The audit emits canonical recipe, manifest, and split-ledger SHA-256 digests. Artifact envelope v3 +can persist those digests with model revision, task heads, hierarchical sampling policy, seed, and +evaluation summary. The loader remains compatible with v2 envelopes; the public eight-emotion +inference schema is unchanged. + +Frozen-encoder experiments can use the sampling primitives in +`ser._internal.models.utterance_sampling`: corpora receive mass proportional to the square root of +their utterance count, classes receive inverse-square-root mass within corpus, and long utterances +use a bounded seeded window subset per epoch. Evaluation should continue aggregating all windows to +one utterance prediction. diff --git a/ser/_internal/data/dataset_audit.py b/ser/_internal/data/dataset_audit.py new file mode 100644 index 0000000..a59c725 --- /dev/null +++ b/ser/_internal/data/dataset_audit.py @@ -0,0 +1,313 @@ +"""Content deduplication and leakage-safe dataset split ledgers.""" + +from __future__ import annotations + +import hashlib +import json +from collections import Counter, defaultdict +from dataclasses import dataclass +from typing import Literal + +from ser._internal.data.manifest import SplitName, Utterance +from ser._internal.data.recipe import ( + DatasetRecipe, + RouteDisposition, + RoutedUtterance, + route_utterance, +) + +type LedgerSplit = Literal["train", "dev", "test", "ssl_only", "quarantined"] + + +class DatasetAuditError(ValueError): + """Raised when a recipe cannot produce a defensible benchmark.""" + + +@dataclass(frozen=True) +class SplitLedgerEntry: + """Immutable split assignment for one manifest row.""" + + sample_id: str + corpus: str + split: LedgerSplit + group_id: str | None + normalized_audio_sha256: str | None + tasks: tuple[str, ...] + disposition: RouteDisposition + reason: str + + def to_record(self) -> dict[str, object]: + """Returns a deterministic JSON-compatible ledger record.""" + return { + "sample_id": self.sample_id, + "corpus": self.corpus, + "split": self.split, + "group_id": self.group_id, + "normalized_audio_sha256": self.normalized_audio_sha256, + "tasks": list(self.tasks), + "disposition": self.disposition, + "reason": self.reason, + } + + +@dataclass(frozen=True) +class DatasetAuditReport: + """Reproducible recipe, manifest, routing, and split audit output.""" + + recipe_id: str + recipe_revision: str + recipe_digest: str + manifest_digest: str + split_ledger_digest: str + seed: int + counters: dict[str, int] + ledger: tuple[SplitLedgerEntry, ...] + + +def _canonical_manifest_digest(utterances: list[Utterance]) -> str: + records: list[dict[str, object]] = [] + for row in sorted(utterances, key=lambda item: item.sample_id): + record = row.to_record() + record.pop("audio_path", None) + records.append(record) + payload = json.dumps(records, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _ledger_digest(entries: list[SplitLedgerEntry]) -> str: + payload = json.dumps( + [entry.to_record() for entry in sorted(entries, key=lambda row: row.sample_id)], + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _identity_components(rows: list[RoutedUtterance]) -> dict[str, str | None]: + parent: dict[str, str] = {} + + def find(value: str) -> str: + parent.setdefault(value, value) + if parent[value] != value: + parent[value] = find(parent[value]) + return parent[value] + + def union(left: str, right: str) -> None: + left_root = find(left) + right_root = find(right) + if left_root != right_root: + parent[max(left_root, right_root)] = min(left_root, right_root) + + for routed in rows: + identities = [ + value + for value in (routed.utterance.speaker_id, routed.utterance.session_id) + if value is not None + ] + if identities: + find(identities[0]) + for identity in identities[1:]: + union(identities[0], identity) + + result: dict[str, str | None] = {} + for routed in rows: + identities = [ + value + for value in (routed.utterance.speaker_id, routed.utterance.session_id) + if value is not None + ] + result[routed.utterance.sample_id] = find(identities[0]) if identities else None + return result + + +def _group_assignments(group_ids: set[str], *, corpus: str, seed: int) -> dict[str, SplitName]: + ordered = sorted( + group_ids, + key=lambda group: hashlib.sha256(f"{seed}:{corpus}:{group}".encode()).digest(), + ) + count = len(ordered) + if count == 1: + return {ordered[0]: "train"} + if count == 2: + return {ordered[0]: "train", ordered[1]: "test"} + train_count = max(1, min(count - 2, round(count * 0.70))) + remaining = count - train_count + dev_count = max(1, min(remaining - 1, round(count * 0.15))) + return { + group: ( + "train" if index < train_count else "dev" if index < train_count + dev_count else "test" + ) + for index, group in enumerate(ordered) + } + + +def _validate_partition_isolation(entries: list[SplitLedgerEntry]) -> None: + supervised = [entry for entry in entries if entry.split in {"train", "dev", "test"}] + for attribute in ("group_id", "normalized_audio_sha256"): + owners: dict[str, LedgerSplit] = {} + for entry in supervised: + value = getattr(entry, attribute) + if value is None: + continue + previous = owners.setdefault(value, entry.split) + if previous != entry.split: + raise DatasetAuditError( + f"Split leakage: {attribute} {value!r} appears in {previous!r} and {entry.split!r}." + ) + + +def audit_dataset_recipe( + utterances: list[Utterance], + *, + recipe: DatasetRecipe, + seed: int = 17, + strict: bool = True, +) -> DatasetAuditReport: + """Audits all rows, deduplicates content, and builds per-corpus split assignments. + + Strict mode rejects missing revisions or hashes, duplicate content, leakage, and a + primary task with fewer than two populated classes. + """ + recipe.validate() + sample_ids: set[str] = set() + content_samples: defaultdict[str, list[str]] = defaultdict(list) + routes: list[RoutedUtterance] = [] + duplicate_ids: set[str] = set() + missing_hash_ids: set[str] = set() + for utterance in utterances: + if utterance.sample_id in sample_ids: + raise DatasetAuditError( + f"Duplicate sample_id {utterance.sample_id!r} across manifests." + ) + sample_ids.add(utterance.sample_id) + content_hash = utterance.normalized_audio_sha256 + if content_hash is None: + missing_hash_ids.add(utterance.sample_id) + else: + content_samples[content_hash].append(utterance.sample_id) + routes.append(route_utterance(utterance, recipe)) + + for sample_group in content_samples.values(): + if len(sample_group) > 1: + duplicate_ids.update(sample_group) + + if strict and duplicate_ids: + raise DatasetAuditError( + f"Duplicate normalized audio content detected for {len(duplicate_ids)} row(s)." + ) + if strict and missing_hash_ids: + raise DatasetAuditError( + f"normalized_audio_sha256 is missing for {len(missing_hash_ids)} row(s)." + ) + if strict: + missing_revisions = [row.sample_id for row in utterances if row.dataset_revision is None] + if missing_revisions: + raise DatasetAuditError( + f"dataset_revision is missing for {len(missing_revisions)} row(s)." + ) + + counters: Counter[str] = Counter(route.disposition for route in routes) + entries: list[SplitLedgerEntry] = [] + by_corpus: dict[str, list[RoutedUtterance]] = defaultdict(list) + for route in routes: + by_corpus[route.utterance.corpus].append(route) + + for corpus, corpus_routes in sorted(by_corpus.items()): + identities = _identity_components(corpus_routes) + eligible = [ + route + for route in corpus_routes + if route.disposition not in {"dropped", "missing", "quarantined"} + and route.utterance.sample_id not in duplicate_ids + ] + official = bool(eligible) and all( + (route.utterance.native_split or route.utterance.split) is not None + for route in eligible + ) + group_ids = { + identity + for route in eligible + if (identity := identities[route.utterance.sample_id]) is not None + } + assignments = ( + _group_assignments(group_ids, corpus=corpus, seed=seed) if not official else {} + ) + + for route in corpus_routes: + utterance = route.utterance + group_id = identities[utterance.sample_id] + if utterance.sample_id in duplicate_ids: + split: LedgerSplit = "quarantined" + reason = "duplicate_normalized_audio" + disposition: RouteDisposition = "quarantined" + counters[route.disposition] -= 1 + counters["quarantined"] += 1 + elif route.disposition in {"dropped", "missing", "quarantined"}: + split = "quarantined" + reason = route.reason + disposition = route.disposition + elif official: + native = utterance.native_split or utterance.split + assert native is not None + split = native + reason = "verified_native_split" + disposition = route.disposition + elif group_id is None: + split = "ssl_only" + reason = "missing_speaker_or_session_group" + disposition = route.disposition + else: + split = assignments[group_id] + reason = "deterministic_grouped_split" + disposition = route.disposition + entries.append( + SplitLedgerEntry( + sample_id=utterance.sample_id, + corpus=utterance.corpus, + split=split, + group_id=group_id, + normalized_audio_sha256=utterance.normalized_audio_sha256, + tasks=tuple(sorted(route.tasks)), + disposition=disposition, + reason=reason, + ) + ) + + if sum(counters.values()) != len(utterances): + raise DatasetAuditError("Internal audit accounting did not classify every manifest row.") + _validate_partition_isolation(entries) + if strict: + utterance_by_id = {utterance.sample_id: utterance for utterance in utterances} + train_labels = { + label + for entry in entries + if entry.split == "train" and "primary_emotion" in entry.tasks + if (label := utterance_by_id[entry.sample_id].label) is not None + } + if len(train_labels) < 2: + raise DatasetAuditError( + "Primary emotion training partition must contain at least two populated classes." + ) + evaluation_labels = { + label + for entry in entries + if entry.split in {"dev", "test"} and "primary_emotion" in entry.tasks + if (label := utterance_by_id[entry.sample_id].label) is not None + } + missing_train_labels = evaluation_labels - train_labels + if missing_train_labels: + raise DatasetAuditError( + "Primary emotion evaluation classes are absent from train: " + + ", ".join(sorted(missing_train_labels)) + ) + + return DatasetAuditReport( + recipe_id=recipe.recipe_id, + recipe_revision=recipe.revision, + recipe_digest=recipe.digest, + manifest_digest=_canonical_manifest_digest(utterances), + split_ledger_digest=_ledger_digest(entries), + seed=seed, + counters=dict(sorted(counters.items())), + ledger=tuple(sorted(entries, key=lambda row: row.sample_id)), + ) diff --git a/ser/_internal/data/recipe.py b/ser/_internal/data/recipe.py new file mode 100644 index 0000000..4ef4d76 --- /dev/null +++ b/ser/_internal/data/recipe.py @@ -0,0 +1,313 @@ +"""Explicit, versioned dataset recipes and task routing.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, cast + +from ser._internal.data.manifest import Utterance +from ser._internal.data.ontology import normalize_label + +DATASET_RECIPE_SCHEMA_VERSION = 1 +CANONICAL_EMOTIONS = frozenset( + {"neutral", "calm", "happy", "sad", "angry", "fearful", "disgust", "surprised"} +) + +type TaskName = Literal[ + "primary_emotion", + "raw_emotion", + "vad", + "attitude", + "binary_affect", + "language", + "text_alignment", + "ssl", +] +type RouteDisposition = Literal["accepted", "remapped", "weak", "dropped", "missing", "quarantined"] +_TASK_NAMES = frozenset( + { + "primary_emotion", + "raw_emotion", + "vad", + "attitude", + "binary_affect", + "language", + "text_alignment", + "ssl", + } +) + + +@dataclass(frozen=True) +class CorpusRecipe: + """Task policy for one corpus.""" + + corpus: str + exact_primary_labels: frozenset[str] = frozenset() + approximate_labels: frozenset[str] = frozenset() + auxiliary_tasks: tuple[TaskName, ...] = () + + def validate(self) -> None: + """Validates one corpus routing policy.""" + if not self.corpus.strip(): + raise ValueError("Corpus recipe id must be non-empty.") + if self.exact_primary_labels - CANONICAL_EMOTIONS: + raise ValueError(f"Corpus {self.corpus!r} contains non-canonical primary labels.") + if self.exact_primary_labels & self.approximate_labels: + raise ValueError(f"Corpus {self.corpus!r} has labels marked exact and approximate.") + if "primary_emotion" in self.auxiliary_tasks: + raise ValueError("primary_emotion must be configured through exact_primary_labels.") + if not set(self.auxiliary_tasks).issubset(_TASK_NAMES): + raise ValueError(f"Corpus {self.corpus!r} contains unsupported auxiliary tasks.") + + def to_record(self) -> dict[str, object]: + """Returns a deterministic JSON-compatible recipe record.""" + return { + "corpus": self.corpus, + "exact_primary_labels": sorted(self.exact_primary_labels), + "approximate_labels": sorted(self.approximate_labels), + "auxiliary_tasks": list(self.auxiliary_tasks), + } + + +@dataclass(frozen=True) +class DatasetRecipe: + """Versioned declaration of corpora, ontology, and training tasks.""" + + recipe_id: str + revision: str + ontology_version: str + corpora: tuple[CorpusRecipe, ...] + schema_version: int = DATASET_RECIPE_SCHEMA_VERSION + + def validate(self) -> None: + """Validates required revisions and unique corpus policies.""" + if self.schema_version != DATASET_RECIPE_SCHEMA_VERSION: + raise ValueError(f"Unsupported dataset recipe schema {self.schema_version!r}.") + for field_name, value in ( + ("recipe_id", self.recipe_id), + ("revision", self.revision), + ("ontology_version", self.ontology_version), + ): + if not value.strip(): + raise ValueError(f"Dataset recipe {field_name} must be non-empty.") + corpus_ids: set[str] = set() + for corpus in self.corpora: + corpus.validate() + if corpus.corpus in corpus_ids: + raise ValueError(f"Duplicate corpus recipe {corpus.corpus!r}.") + corpus_ids.add(corpus.corpus) + if not corpus_ids: + raise ValueError("Dataset recipe must include at least one corpus.") + + def to_record(self) -> dict[str, object]: + """Returns the canonical JSON-compatible representation.""" + return { + "schema_version": self.schema_version, + "recipe_id": self.recipe_id, + "revision": self.revision, + "ontology_version": self.ontology_version, + "corpora": [ + corpus.to_record() for corpus in sorted(self.corpora, key=lambda row: row.corpus) + ], + } + + @property + def digest(self) -> str: + """Returns the SHA-256 digest of the canonical recipe representation.""" + self.validate() + payload = json.dumps(self.to_record(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + def corpus_policy(self, corpus: str) -> CorpusRecipe | None: + """Returns the policy for one corpus, if declared.""" + return next((policy for policy in self.corpora if policy.corpus == corpus), None) + + +@dataclass(frozen=True) +class RoutedUtterance: + """Exhaustive routing result for one manifest row.""" + + utterance: Utterance + disposition: RouteDisposition + tasks: frozenset[TaskName] + reason: str + + +def route_utterance(utterance: Utterance, recipe: DatasetRecipe) -> RoutedUtterance: + """Routes one row without forcing incompatible labels into the primary head.""" + policy = recipe.corpus_policy(utterance.corpus) + if policy is None: + return RoutedUtterance(utterance, "quarantined", frozenset(), "corpus_not_in_recipe") + + tasks: set[TaskName] = {"ssl"} + if utterance.vad is not None and "vad" in policy.auxiliary_tasks: + tasks.add("vad") + label = utterance.label + if ( + utterance.social_attitude is not None or label is not None + ) and "attitude" in policy.auxiliary_tasks: + tasks.add("attitude") + if ( + utterance.binary_affect is not None or label is not None + ) and "binary_affect" in policy.auxiliary_tasks: + tasks.add("binary_affect") + if utterance.language is not None and "language" in policy.auxiliary_tasks: + tasks.add("language") + if utterance.transcript is not None and "text_alignment" in policy.auxiliary_tasks: + tasks.add("text_alignment") + + raw_label = normalize_label(utterance.raw_label) if utterance.raw_label else label + if ( + label is not None + and label in policy.exact_primary_labels + and raw_label not in policy.approximate_labels + ): + tasks.add("primary_emotion") + disposition: RouteDisposition = "remapped" if raw_label != label else "accepted" + return RoutedUtterance(utterance, disposition, frozenset(tasks), "exact_primary_label") + if raw_label is not None and raw_label in policy.approximate_labels: + tasks.add("raw_emotion") + return RoutedUtterance( + utterance, + "weak", + frozenset(tasks), + "approximate_label_is_auxiliary_only", + ) + if label is not None and "raw_emotion" in policy.auxiliary_tasks: + tasks.add("raw_emotion") + if len(tasks) > 1: + return RoutedUtterance(utterance, "accepted", frozenset(tasks), "auxiliary_targets") + if label is None and all( + target is None + for target in ( + utterance.vad, + utterance.social_attitude, + utterance.binary_affect, + utterance.language, + utterance.transcript, + ) + ): + return RoutedUtterance(utterance, "missing", frozenset(tasks), "no_usable_targets") + return RoutedUtterance(utterance, "dropped", frozenset(tasks), "target_not_enabled_by_recipe") + + +def research_recipe_v1() -> DatasetRecipe: + """Returns the common leakage-safe cross-domain research recipe.""" + exact_corpora = ( + "ravdess", + "crema-d", + "msp-podcast", + "mesd", + "oreau-french-esd", + "cafe", + "asvp-esd", + "spanish-meacorpus-2023", + "biic-podcast", + ) + policies = [ + CorpusRecipe(corpus=corpus, exact_primary_labels=CANONICAL_EMOTIONS) + for corpus in exact_corpora + ] + policies.extend( + ( + CorpusRecipe( + corpus="escorpus-pe", + approximate_labels=frozenset({"boredom", "neutral"}), + auxiliary_tasks=("vad", "language"), + ), + CorpusRecipe( + corpus="att-hack", + auxiliary_tasks=("attitude", "language", "text_alignment"), + ), + CorpusRecipe( + corpus="coraa-ser", + auxiliary_tasks=("binary_affect", "language", "text_alignment"), + ), + CorpusRecipe( + corpus="emodb-2.0", + exact_primary_labels=CANONICAL_EMOTIONS, + approximate_labels=frozenset({"boredom"}), + ), + CorpusRecipe( + corpus="emov-db", + exact_primary_labels=CANONICAL_EMOTIONS, + approximate_labels=frozenset({"anxious", "amused", "sleepy"}), + ), + CorpusRecipe(corpus="pavoque", auxiliary_tasks=("raw_emotion", "language")), + CorpusRecipe(corpus="jl-corpus", exact_primary_labels=CANONICAL_EMOTIONS), + ) + ) + return DatasetRecipe( + recipe_id="cross-domain-common", + revision="1", + ontology_version="canonical-eight-v1", + corpora=tuple(policies), + ) + + +def load_dataset_recipe(value: str | Path) -> DatasetRecipe: + """Loads a built-in recipe id or a versioned JSON recipe file.""" + if str(value) == "research-v1": + return research_recipe_v1() + path = Path(value).expanduser() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as err: + raise ValueError(f"Unable to load dataset recipe {path}: {err}") from err + if not isinstance(payload, dict): + raise ValueError("Dataset recipe root must be a JSON object.") + corpora_raw = payload.get("corpora") + if not isinstance(corpora_raw, list): + raise ValueError("Dataset recipe 'corpora' must be a list.") + corpora: list[CorpusRecipe] = [] + for raw in corpora_raw: + if not isinstance(raw, dict): + raise ValueError("Dataset recipe corpora must contain objects.") + corpus = raw.get("corpus") + exact = raw.get("exact_primary_labels", []) + approximate = raw.get("approximate_labels", []) + tasks = raw.get("auxiliary_tasks", []) + if not isinstance(corpus, str) or not corpus.strip(): + raise ValueError("Dataset recipe corpus id must be non-empty.") + for field_name, field_value in ( + ("exact_primary_labels", exact), + ("approximate_labels", approximate), + ("auxiliary_tasks", tasks), + ): + if not isinstance(field_value, list) or any( + not isinstance(item, str) or not item.strip() for item in field_value + ): + raise ValueError(f"Dataset recipe {field_name!r} must be a list of strings.") + corpora.append( + CorpusRecipe( + corpus=corpus.strip(), + exact_primary_labels=frozenset(normalize_label(item) for item in exact), + approximate_labels=frozenset(normalize_label(item) for item in approximate), + auxiliary_tasks=tuple(cast(TaskName, item.strip()) for item in tasks), + ) + ) + schema_version = payload.get("schema_version") + recipe_id = payload.get("recipe_id") + revision = payload.get("revision") + ontology_version = payload.get("ontology_version") + if ( + not isinstance(schema_version, int) + or isinstance(schema_version, bool) + or not isinstance(recipe_id, str) + or not isinstance(revision, str) + or not isinstance(ontology_version, str) + ): + raise ValueError("Dataset recipe is missing required schema/id/revision/ontology fields.") + recipe = DatasetRecipe( + schema_version=schema_version, + recipe_id=recipe_id, + revision=revision, + ontology_version=ontology_version, + corpora=tuple(corpora), + ) + recipe.validate() + return recipe diff --git a/tests/suites/unit/data/test_dataset_recipe_audit.py b/tests/suites/unit/data/test_dataset_recipe_audit.py new file mode 100644 index 0000000..fc1a6de --- /dev/null +++ b/tests/suites/unit/data/test_dataset_recipe_audit.py @@ -0,0 +1,157 @@ +"""Tests for explicit task routing and leakage-safe dataset audits.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import pytest + +from ser._internal.data.dataset_audit import DatasetAuditError, audit_dataset_recipe +from ser._internal.data.manifest import MANIFEST_SCHEMA_VERSION, Utterance, VadTarget +from ser._internal.data.recipe import CorpusRecipe, DatasetRecipe, route_utterance + + +def _recipe() -> DatasetRecipe: + return DatasetRecipe( + recipe_id="fixture", + revision="1", + ontology_version="canonical-eight-v1", + corpora=( + CorpusRecipe( + corpus="acted", + exact_primary_labels=frozenset({"happy", "sad", "neutral"}), + approximate_labels=frozenset({"boredom"}), + ), + CorpusRecipe(corpus="vad", auxiliary_tasks=("vad",)), + ), + ) + + +def _row( + sample_id: str, + *, + label: str | None, + speaker: str | None, + digest: str, + raw_label: str | None = None, + corpus: str = "acted", +) -> Utterance: + return Utterance( + schema_version=MANIFEST_SCHEMA_VERSION, + sample_id=sample_id, + corpus=corpus, + audio_path=Path(f"{sample_id}.wav"), + label=label, + raw_label=raw_label, + vad=VadTarget(0.0, 0.2, -0.1) if corpus == "vad" else None, + speaker_id=speaker, + normalized_audio_sha256=digest, + dataset_revision="fixture-r1", + ) + + +def test_exact_and_approximate_labels_route_to_isolated_heads() -> None: + """Approximate labels never train the public primary emotion head.""" + exact = route_utterance( + _row("acted:exact", label="happy", speaker="acted:s1", digest="1" * 64), + _recipe(), + ) + weak = route_utterance( + _row( + "acted:weak", + label="neutral", + raw_label="boredom", + speaker="acted:s2", + digest="2" * 64, + ), + _recipe(), + ) + + assert exact.disposition == "accepted" + assert "primary_emotion" in exact.tasks + assert weak.disposition == "weak" + assert "raw_emotion" in weak.tasks + assert "primary_emotion" not in weak.tasks + + +def test_audit_is_deterministic_grouped_and_exhaustively_accounted() -> None: + """Per-corpus grouping is deterministic and every input receives one disposition.""" + rows = [ + _row( + f"acted:{index}", + label="happy" if index % 2 else "sad", + speaker=f"acted:s{index}", + digest=f"{index:064x}", + ) + for index in range(1, 7) + ] + rows.append(_row("vad:1", label=None, speaker=None, digest="f" * 64, corpus="vad")) + + first = audit_dataset_recipe(rows, recipe=_recipe(), seed=9) + second = audit_dataset_recipe(list(reversed(rows)), recipe=_recipe(), seed=9) + + assert first.split_ledger_digest == second.split_ledger_digest + assert sum(first.counters.values()) == len(rows) + vad_entry = next(entry for entry in first.ledger if entry.sample_id == "vad:1") + assert vad_entry.split == "ssl_only" + assert {entry.split for entry in first.ledger if entry.corpus == "acted"} == { + "train", + "dev", + "test", + } + + +def test_audit_rejects_duplicate_content_before_splitting() -> None: + """Identical normalized PCM cannot cross sample or corpus boundaries.""" + left = _row("acted:1", label="happy", speaker="acted:s1", digest="a" * 64) + right = replace(left, sample_id="acted:2", speaker_id="acted:s2", label="sad") + + with pytest.raises(DatasetAuditError, match="Duplicate normalized audio"): + audit_dataset_recipe([left, right], recipe=_recipe()) + + +def test_non_strict_audit_quarantines_every_duplicate_deterministically() -> None: + """Exploratory audits cannot choose a duplicate survivor based on input order.""" + left = _row("acted:1", label="happy", speaker="acted:s1", digest="a" * 64) + right = replace(left, sample_id="acted:2", speaker_id="acted:s2", label="sad") + + first = audit_dataset_recipe([left, right], recipe=_recipe(), strict=False) + second = audit_dataset_recipe([right, left], recipe=_recipe(), strict=False) + + assert first.split_ledger_digest == second.split_ledger_digest + assert {entry.split for entry in first.ledger} == {"quarantined"} + + +def test_manifest_digest_is_independent_of_install_root() -> None: + """Moving identical content does not alter the reproducibility digest.""" + rows = [ + _row( + f"acted:{index}", + label="happy" if index % 2 else "sad", + speaker=f"acted:s{index}", + digest=f"{index:064x}", + ) + for index in range(1, 7) + ] + moved = [replace(row, audio_path=Path("/other/root") / row.audio_path.name) for row in rows] + + first = audit_dataset_recipe(rows, recipe=_recipe(), seed=9) + second = audit_dataset_recipe(moved, recipe=_recipe(), seed=9) + + assert first.manifest_digest == second.manifest_digest + + +def test_audit_rejects_official_identity_leakage() -> None: + """Official split labels do not override speaker/session leakage checks.""" + left = replace( + _row("acted:1", label="happy", speaker="acted:s1", digest="a" * 64), + native_split="train", + ) + right = replace( + _row("acted:2", label="sad", speaker="acted:s1", digest="b" * 64), + native_split="test", + ) + + with pytest.raises(DatasetAuditError, match="Split leakage"): + audit_dataset_recipe([left, right], recipe=_recipe()) From db8a0e46eeeb876dae9aa4a1d7d6eb0fe1bf604a Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:18:39 -0300 Subject: [PATCH 03/10] feat(config): add recipe, quarantine, and dev-split settings Adds the settings the recipe and readiness work reads: dataset recipe selection and strict auditing, bias-aware quarantine budgets, and a development split size. Strict auditing defaults on whenever a recipe is configured, since a recipe exists to produce a defensible benchmark. The quarantine budgets are deliberately several narrow limits rather than one global ratio: a single ratio can be satisfied while silently deleting one corpus or one class. Public API surface additions are config attributes only. --- ser/_internal/config/schema.py | 9 ++ ser/_internal/config/settings_builder.py | 9 ++ ser/_internal/config/settings_inputs.py | 43 +++++++++ tests/suites/integration/api/test_api.py | 8 +- .../architecture/public_api_snapshot.json | 90 +++++++++++++++++++ 5 files changed, 156 insertions(+), 3 deletions(-) diff --git a/ser/_internal/config/schema.py b/ser/_internal/config/schema.py index 773a1db..fd41edf 100644 --- a/ser/_internal/config/schema.py +++ b/ser/_internal/config/schema.py @@ -256,6 +256,8 @@ class DatasetConfig: subfolder_prefix: str = "Actor_*" extension: str = "*.wav" manifest_paths: tuple[Path, ...] = () + recipe: str | None = None + strict_audit: bool = False @property def glob_pattern(self) -> str: @@ -269,6 +271,12 @@ class DataLoaderConfig: max_workers: int = 8 max_failed_file_ratio: float = 0.01 + max_failed_files: int = 25 + max_failed_file_ratio_per_corpus: float = 0.01 + max_failed_file_ratio_per_class: float = 0.01 + max_failures_per_reason: int = 10 + min_remaining_per_class_split: int = 1 + strict_quarantine: bool = False @dataclass(frozen=True) @@ -276,6 +284,7 @@ class TrainingConfig: """Dataset split controls for model training.""" test_size: float = 0.25 + dev_size: float = 0.10 random_state: int = 42 stratify_split: bool = True diff --git a/ser/_internal/config/settings_builder.py b/ser/_internal/config/settings_builder.py index 952c8f3..790c35e 100644 --- a/ser/_internal/config/settings_builder.py +++ b/ser/_internal/config/settings_builder.py @@ -87,13 +87,22 @@ def build_settings_from_inputs( dataset=config_schema.DatasetConfig( folder=inputs.dataset_folder, manifest_paths=inputs.manifest_paths, + recipe=inputs.dataset_recipe, + strict_audit=inputs.strict_dataset_audit, ), data_loader=config_schema.DataLoaderConfig( max_workers=inputs.max_workers, max_failed_file_ratio=inputs.max_failed_file_ratio, + max_failed_files=inputs.max_failed_files, + max_failed_file_ratio_per_corpus=inputs.max_failed_file_ratio_per_corpus, + max_failed_file_ratio_per_class=inputs.max_failed_file_ratio_per_class, + max_failures_per_reason=inputs.max_failures_per_reason, + min_remaining_per_class_split=inputs.min_remaining_per_class_split, + strict_quarantine=inputs.strict_quarantine, ), training=config_schema.TrainingConfig( test_size=inputs.test_size, + dev_size=inputs.dev_size, random_state=inputs.random_state, ), models=config_schema.ModelsConfig( diff --git a/ser/_internal/config/settings_inputs.py b/ser/_internal/config/settings_inputs.py index 194070c..9b3bb4e 100644 --- a/ser/_internal/config/settings_inputs.py +++ b/ser/_internal/config/settings_inputs.py @@ -27,10 +27,19 @@ class ResolvedSettingsInputs: dataset_folder: Path manifest_paths: tuple[Path, ...] + dataset_recipe: str | None + strict_dataset_audit: bool default_language: str max_workers: int max_failed_file_ratio: float + max_failed_files: int + max_failed_file_ratio_per_corpus: float + max_failed_file_ratio_per_class: float + max_failures_per_reason: int + min_remaining_per_class_split: int + strict_quarantine: bool test_size: float + dev_size: float random_state: int profile_pipeline: bool medium_profile: bool @@ -165,6 +174,12 @@ def resolve_settings_inputs(deps: SettingsInputDeps) -> ResolvedSettingsInputs: manifest_paths = deps.parse_manifest_paths( (deps.getenv("SER_DATASET_MANIFESTS", "") or "").strip() ) + dataset_recipe_raw = (deps.getenv("SER_DATASET_RECIPE", "") or "").strip() + dataset_recipe = dataset_recipe_raw or None + strict_dataset_audit = deps.read_bool_env( + "SER_STRICT_DATASET_AUDIT", + dataset_recipe is not None, + ) default_language = deps.getenv("DEFAULT_LANGUAGE", "en") or "en" max_workers = deps.read_int_env("SER_MAX_WORKERS", 8, minimum=1) @@ -174,7 +189,26 @@ def resolve_settings_inputs(deps: SettingsInputDeps) -> ResolvedSettingsInputs: minimum=0.0, maximum=1.0, ) + max_failed_files = deps.read_int_env("SER_MAX_FAILED_FILES", 25, minimum=0) + max_failed_file_ratio_per_corpus = deps.read_float_env( + "SER_MAX_FAILED_FILE_RATIO_PER_CORPUS", + max_failed_file_ratio, + minimum=0.0, + maximum=1.0, + ) + max_failed_file_ratio_per_class = deps.read_float_env( + "SER_MAX_FAILED_FILE_RATIO_PER_CLASS", + max_failed_file_ratio, + minimum=0.0, + maximum=1.0, + ) + max_failures_per_reason = deps.read_int_env("SER_MAX_FAILURES_PER_REASON", 10, minimum=0) + min_remaining_per_class_split = deps.read_int_env( + "SER_MIN_REMAINING_PER_CLASS_SPLIT", 1, minimum=0 + ) + strict_quarantine = deps.read_bool_env("SER_STRICT_QUARANTINE", False) test_size = deps.read_float_env("SER_TEST_SIZE", 0.25, minimum=0.05, maximum=0.95) + dev_size = deps.read_float_env("SER_DEV_SIZE", 0.10, minimum=0.0, maximum=0.5) random_state = deps.read_int_env("SER_RANDOM_STATE", 42, minimum=0) profile_catalog = deps.get_profile_catalog() @@ -354,10 +388,19 @@ def resolve_settings_inputs(deps: SettingsInputDeps) -> ResolvedSettingsInputs: return ResolvedSettingsInputs( dataset_folder=dataset_folder, manifest_paths=manifest_paths, + dataset_recipe=dataset_recipe, + strict_dataset_audit=strict_dataset_audit, default_language=default_language, max_workers=max_workers, max_failed_file_ratio=max_failed_file_ratio, + max_failed_files=max_failed_files, + max_failed_file_ratio_per_corpus=max_failed_file_ratio_per_corpus, + max_failed_file_ratio_per_class=max_failed_file_ratio_per_class, + max_failures_per_reason=max_failures_per_reason, + min_remaining_per_class_split=min_remaining_per_class_split, + strict_quarantine=strict_quarantine, test_size=test_size, + dev_size=dev_size, random_state=random_state, profile_pipeline=profile_pipeline, medium_profile=medium_profile, diff --git a/tests/suites/integration/api/test_api.py b/tests/suites/integration/api/test_api.py index 5f4842c..f141f0a 100644 --- a/tests/suites/integration/api/test_api.py +++ b/tests/suites/integration/api/test_api.py @@ -176,6 +176,8 @@ def test_run_transcription_runtime_calibration_command_maps_unexpected_errors( assert disposition.exit_code == 1 assert disposition.message == "Transcription runtime calibration failed: boom" assert disposition.include_traceback is True + assert isinstance(disposition.exception, RuntimeError) + assert isinstance(disposition.exc_info, tuple) def test_run_training_command_maps_training_exceptions_to_disposition( @@ -193,9 +195,9 @@ def test_run_training_command_maps_training_exceptions_to_disposition( ) assert disposition is not None - assert disposition.exit_code == 2 - assert disposition.message == "training failed" - assert disposition.include_traceback is False + assert disposition.exit_code == 1 + assert disposition.message == "Training workflow failed: training failed" + assert disposition.include_traceback is True def test_run_training_command_delegates_arguments_on_success( diff --git a/tests/suites/integration/architecture/public_api_snapshot.json b/tests/suites/integration/architecture/public_api_snapshot.json index 4607b04..c29ce71 100644 --- a/tests/suites/integration/architecture/public_api_snapshot.json +++ b/tests/suites/integration/architecture/public_api_snapshot.json @@ -375,10 +375,40 @@ "kind": "attribute", "value": "0.01" }, + "max_failed_file_ratio_per_class": { + "annotation": "float", + "kind": "attribute", + "value": "0.01" + }, + "max_failed_file_ratio_per_corpus": { + "annotation": "float", + "kind": "attribute", + "value": "0.01" + }, + "max_failed_files": { + "annotation": "int", + "kind": "attribute", + "value": "25" + }, + "max_failures_per_reason": { + "annotation": "int", + "kind": "attribute", + "value": "10" + }, "max_workers": { "annotation": "int", "kind": "attribute", "value": "8" + }, + "min_remaining_per_class_split": { + "annotation": "int", + "kind": "attribute", + "value": "1" + }, + "strict_quarantine": { + "annotation": "bool", + "kind": "attribute", + "value": "False" } } }, @@ -409,6 +439,16 @@ "kind": "attribute", "value": "()" }, + "recipe": { + "annotation": "str | None", + "kind": "attribute", + "value": "None" + }, + "strict_audit": { + "annotation": "bool", + "kind": "attribute", + "value": "False" + }, "subfolder_prefix": { "annotation": "str", "kind": "attribute", @@ -1364,6 +1404,11 @@ "bases": [], "kind": "class", "members": { + "dev_size": { + "annotation": "float", + "kind": "attribute", + "value": "0.1" + }, "random_state": { "annotation": "int", "kind": "attribute", @@ -2083,10 +2128,40 @@ "kind": "attribute", "value": "0.01" }, + "max_failed_file_ratio_per_class": { + "annotation": "float", + "kind": "attribute", + "value": "0.01" + }, + "max_failed_file_ratio_per_corpus": { + "annotation": "float", + "kind": "attribute", + "value": "0.01" + }, + "max_failed_files": { + "annotation": "int", + "kind": "attribute", + "value": "25" + }, + "max_failures_per_reason": { + "annotation": "int", + "kind": "attribute", + "value": "10" + }, "max_workers": { "annotation": "int", "kind": "attribute", "value": "8" + }, + "min_remaining_per_class_split": { + "annotation": "int", + "kind": "attribute", + "value": "1" + }, + "strict_quarantine": { + "annotation": "bool", + "kind": "attribute", + "value": "False" } } }, @@ -2117,6 +2192,16 @@ "kind": "attribute", "value": "()" }, + "recipe": { + "annotation": "str | None", + "kind": "attribute", + "value": "None" + }, + "strict_audit": { + "annotation": "bool", + "kind": "attribute", + "value": "False" + }, "subfolder_prefix": { "annotation": "str", "kind": "attribute", @@ -2676,6 +2761,11 @@ "bases": [], "kind": "class", "members": { + "dev_size": { + "annotation": "float", + "kind": "attribute", + "value": "0.1" + }, "random_state": { "annotation": "int", "kind": "attribute", From 923a31fae7cdb856c2ffc4306b04f8fcc5d685f3 Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:18:39 -0300 Subject: [PATCH 04/10] perf(models): partition speakers in linear time and split three ways split_utterances resolved partition membership with `utterance in `, making speaker partitioning quadratic. Across the ~152k-utterance inventory this extrapolated to roughly nine hours and read as a hang rather than as slow code. Membership now tests against a hashed set. Also adds split_utterances_three_way, which preserves complete native train/dev/test assignments when a manifest carries them and otherwise reserves a deterministic, speaker-isolated development partition. --- ser/_internal/models/dataset_splitting.py | 128 ++++++++++++++---- .../unit/models/test_dataset_splitting.py | 118 ++++++++++++++++ 2 files changed, 223 insertions(+), 23 deletions(-) create mode 100644 tests/suites/unit/models/test_dataset_splitting.py diff --git a/ser/_internal/models/dataset_splitting.py b/ser/_internal/models/dataset_splitting.py index 46431b1..8305359 100644 --- a/ser/_internal/models/dataset_splitting.py +++ b/ser/_internal/models/dataset_splitting.py @@ -4,7 +4,8 @@ import logging import os -from dataclasses import dataclass +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace from hashlib import sha1 import numpy as np @@ -28,6 +29,38 @@ class MediumSplitMetadata: speaker_overlap_count: int +def medium_split_metadata_from_mapping(raw: Mapping[str, object]) -> MediumSplitMetadata: + """Validates and restores split metadata from a prepared-plan payload.""" + split_strategy = raw.get("split_strategy") + speaker_grouped = raw.get("speaker_grouped") + speaker_id_coverage = raw.get("speaker_id_coverage") + train_unique_speakers = raw.get("train_unique_speakers") + test_unique_speakers = raw.get("test_unique_speakers") + speaker_overlap_count = raw.get("speaker_overlap_count") + if not isinstance(split_strategy, str) or not split_strategy: + raise ValueError("Prepared split metadata requires a non-empty split_strategy.") + if not isinstance(speaker_grouped, bool): + raise ValueError("Prepared split metadata speaker_grouped must be boolean.") + if not isinstance(speaker_id_coverage, int | float) or isinstance(speaker_id_coverage, bool): + raise ValueError("Prepared split metadata speaker_id_coverage must be numeric.") + if any( + not isinstance(value, int) or isinstance(value, bool) + for value in (train_unique_speakers, test_unique_speakers, speaker_overlap_count) + ): + raise ValueError("Prepared split metadata speaker counts must be integers.") + assert isinstance(train_unique_speakers, int) + assert isinstance(test_unique_speakers, int) + assert isinstance(speaker_overlap_count, int) + return MediumSplitMetadata( + split_strategy=split_strategy, + speaker_grouped=speaker_grouped, + speaker_id_coverage=float(speaker_id_coverage), + train_unique_speakers=train_unique_speakers, + test_unique_speakers=test_unique_speakers, + speaker_overlap_count=speaker_overlap_count, + ) + + def split_labeled_audio_samples( *, samples: list[LabeledAudioSample], @@ -162,7 +195,7 @@ def hash_stratified_split( """Builds deterministic per-label train/test split ordered by salted hash.""" by_label: dict[str, list[Utterance]] = {} for utterance in samples: - by_label.setdefault(utterance.label, []).append(utterance) + by_label.setdefault(utterance.require_label(), []).append(utterance) train: list[Utterance] = [] test: list[Utterance] = [] @@ -199,6 +232,25 @@ def hash_stratified_split( return train, test +def _partition_speakers( + partition: Sequence[Utterance], + *, + samples: Sequence[Utterance], + speaker_ids: Sequence[str | None], +) -> set[str]: + """Returns the distinct resolved speakers within one partition in linear time. + + Membership is tested against a hashed set so large inventories stay O(n); a + plain ``utterance in partition_list`` scan would be O(n**2) over the samples. + """ + members = set(partition) + return { + speaker + for utterance, speaker in zip(samples, speaker_ids, strict=False) + if utterance in members and speaker is not None + } + + def split_utterances( *, samples: list[Utterance], @@ -209,7 +261,7 @@ def split_utterances( if len(samples) < 2: raise RuntimeError("Training requires at least two labeled audio files.") - labels: list[str] = [utterance.label for utterance in samples] + labels: list[str] = [utterance.require_label() for utterance in samples] speaker_ids: list[str | None] = [ resolve_corpus_scoped_speaker_id(utterance) for utterance in samples ] @@ -221,16 +273,12 @@ def split_utterances( train_split = [utterance for utterance in samples if utterance.split in {"train", "dev"}] test_split = [utterance for utterance in samples if utterance.split == "test"] if train_split and test_split: - train_speakers = { - speaker - for utterance, speaker in zip(samples, speaker_ids, strict=False) - if utterance in train_split and speaker is not None - } - test_speakers = { - speaker - for utterance, speaker in zip(samples, speaker_ids, strict=False) - if utterance in test_split and speaker is not None - } + train_speakers = _partition_speakers( + train_split, samples=samples, speaker_ids=speaker_ids + ) + test_speakers = _partition_speakers( + test_split, samples=samples, speaker_ids=speaker_ids + ) return ( train_split, test_split, @@ -300,16 +348,8 @@ def split_utterances( ) if not train_split or not test_split: raise RuntimeError("Deterministic split produced an empty partition; adjust test_size.") - train_speakers = { - speaker - for utterance, speaker in zip(samples, speaker_ids, strict=False) - if utterance in train_split and speaker is not None - } - test_speakers = { - speaker - for utterance, speaker in zip(samples, speaker_ids, strict=False) - if utterance in test_split and speaker is not None - } + train_speakers = _partition_speakers(train_split, samples=samples, speaker_ids=speaker_ids) + test_speakers = _partition_speakers(test_split, samples=samples, speaker_ids=speaker_ids) return ( train_split, test_split, @@ -322,3 +362,45 @@ def split_utterances( speaker_overlap_count=len(train_speakers.intersection(test_speakers)), ), ) + + +def split_utterances_three_way( + *, + samples: list[Utterance], + settings: AppConfig, + logger: logging.Logger, +) -> tuple[list[Utterance], list[Utterance], list[Utterance], MediumSplitMetadata]: + """Returns canonical train/dev/test partitions, preserving complete native assignments.""" + if samples and all(item.split is not None for item in samples): + train = [item for item in samples if item.split == "train"] + dev = [item for item in samples if item.split == "dev"] + test = [item for item in samples if item.split == "test"] + if train and dev and test: + _, _, metadata = split_utterances(samples=samples, settings=settings, logger=logger) + return train, dev, test, replace(metadata, split_strategy="manifest_three_way_split") + initial_train, test, metadata = split_utterances( + samples=samples, settings=settings, logger=logger + ) + relative_dev_size = settings.training.dev_size / (1.0 - settings.training.test_size) + dev_settings = replace( + settings, + training=replace(settings.training, test_size=relative_dev_size), + ) + train, dev, dev_metadata = split_utterances( + samples=initial_train, + settings=dev_settings, + logger=logger, + ) + return ( + train, + dev, + test, + replace( + metadata, + split_strategy=f"{metadata.split_strategy}+dev", + speaker_overlap_count=max( + metadata.speaker_overlap_count, + dev_metadata.speaker_overlap_count, + ), + ), + ) diff --git a/tests/suites/unit/models/test_dataset_splitting.py b/tests/suites/unit/models/test_dataset_splitting.py new file mode 100644 index 0000000..5815bbb --- /dev/null +++ b/tests/suites/unit/models/test_dataset_splitting.py @@ -0,0 +1,118 @@ +"""Regressions for deterministic utterance splitting, including O(n) speaker accounting.""" + +from __future__ import annotations + +import logging +import time +from pathlib import Path + +from ser._internal.config.schema import DatasetConfig, TrainingConfig +from ser._internal.data.manifest import MANIFEST_SCHEMA_VERSION, Utterance +from ser._internal.models.dataset_splitting import ( + _partition_speakers, + split_utterances_three_way, +) +from ser.config import AppConfig + +logger = logging.getLogger(__name__) + + +def _settings(tmp_path: Path, *, test_size: float = 0.25, dev_size: float = 0.20) -> AppConfig: + return AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + training=TrainingConfig(test_size=test_size, dev_size=dev_size), + ) + + +def _utterance(index: int, label: str, *, speaker: str | None = None) -> Utterance: + """Builds an utterance with no manifest split so the hash-stratified path runs.""" + return Utterance( + schema_version=MANIFEST_SCHEMA_VERSION, + sample_id=f"fixture:{index}", + corpus="fixture", + audio_path=Path(f"{index}.wav"), + label=label, + speaker_id=speaker, + split=None, + ) + + +def test_hash_stratified_three_way_split_is_complete_disjoint_and_deterministic( + tmp_path: Path, +) -> None: + """Partial speaker coverage forces the hash path; partitions must stay well-formed.""" + settings = _settings(tmp_path) + # Half the samples carry a speaker id -> coverage < 100% -> hash-stratified path. + samples = [ + _utterance( + index, + "calm" if index % 2 == 0 else "happy", + speaker=f"spk-{index % 5}" if index % 2 == 0 else None, + ) + for index in range(60) + ] + + train, dev, test, metadata = split_utterances_three_way( + samples=list(samples), settings=settings, logger=logger + ) + + assert metadata.split_strategy == "hash_stratified_split+dev" + # Complete and disjoint partition over the original identities. + partition_ids = [item.sample_id for item in (*train, *dev, *test)] + assert sorted(partition_ids) == sorted(item.sample_id for item in samples) + assert len(partition_ids) == len(set(partition_ids)) + assert train and dev and test + + # Deterministic across repeated invocations. + train2, dev2, test2, metadata2 = split_utterances_three_way( + samples=list(samples), settings=settings, logger=logger + ) + assert [u.sample_id for u in train2] == [u.sample_id for u in train] + assert [u.sample_id for u in dev2] == [u.sample_id for u in dev] + assert [u.sample_id for u in test2] == [u.sample_id for u in test] + assert metadata2 == metadata + + +def test_partition_speakers_matches_naive_membership() -> None: + """The set-based helper must equal the original O(n**2) list-membership result.""" + samples = [ + _utterance(0, "calm", speaker="a"), + _utterance(1, "happy", speaker="b"), + _utterance(2, "calm", speaker="a"), # duplicate speaker + _utterance(3, "happy", speaker=None), # no speaker -> excluded + _utterance(4, "calm", speaker="c"), + ] + speaker_ids = [item.speaker_id for item in samples] + partition = [samples[0], samples[2], samples[3]] # speakers a, a, None + + expected = { + speaker + for utterance, speaker in zip(samples, speaker_ids, strict=False) + if utterance in partition and speaker is not None + } + result = _partition_speakers(partition, samples=samples, speaker_ids=speaker_ids) + + assert result == expected == {"a"} + + +def test_three_way_split_scales_linearly_on_large_inventory(tmp_path: Path) -> None: + """Guards against reintroducing O(n**2) membership: 20k rows must finish quickly. + + The prior list-membership implementation took minutes at this size (~n**2 equality + checks); the linear implementation completes in well under a second. The 30s bound + is a wide margin that only a quadratic regression can exceed. + """ + settings = _settings(tmp_path) + samples = [_utterance(index, "calm" if index % 2 == 0 else "happy") for index in range(20_000)] + + started = time.perf_counter() + train, dev, test, metadata = split_utterances_three_way( + samples=list(samples), settings=settings, logger=logger + ) + elapsed = time.perf_counter() - started + + assert metadata.split_strategy == "hash_stratified_split+dev" + assert len(train) + len(dev) + len(test) == len(samples) + assert elapsed < 30.0, f"three-way split took {elapsed:.1f}s; expected linear-time completion" From ca1e268d773f3130ba2c8b356e81d7da1d58ecde Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:18:49 -0300 Subject: [PATCH 05/10] feat(models): add artifact envelope v3 provenance Persists what a run was actually trained from: recipe and split-ledger digests, model revision, task heads, sampling policy, seed, and evaluation summary. All are validated on read, so a malformed digest or a non-JSON policy is rejected rather than silently trusted. The loader accepts both v2 and v3 envelopes and the public eight-emotion inference schema is unchanged. Newly written artifacts are v3, so an older release cannot read them; reading existing v2 artifacts is unaffected. --- ser/_internal/models/artifact_envelope.py | 45 +++++-- ser/_internal/models/artifact_loading.py | 8 +- ser/_internal/models/artifact_metadata.py | 117 ++++++++++++++++++ .../unit/models/test_artifact_upgrade_path.py | 61 +++++++++ 4 files changed, 222 insertions(+), 9 deletions(-) diff --git a/ser/_internal/models/artifact_envelope.py b/ser/_internal/models/artifact_envelope.py index 7a0d73a..f87cec3 100644 --- a/ser/_internal/models/artifact_envelope.py +++ b/ser/_internal/models/artifact_envelope.py @@ -12,13 +12,15 @@ deserialize_model_artifact_envelope, ) from ser._internal.models.artifact_metadata import ( - build_v2_artifact_metadata, + build_v3_artifact_metadata, normalize_v2_artifact_metadata, + normalize_v3_artifact_metadata, read_positive_int, ) from ser.runtime.schema import ARTIFACT_SCHEMA_VERSION -MODEL_ARTIFACT_VERSION = 2 +MODEL_ARTIFACT_VERSION = 3 +SUPPORTED_MODEL_ARTIFACT_VERSIONS = frozenset({2, MODEL_ARTIFACT_VERSION}) DEFAULT_BACKEND_ID = "handcrafted" DEFAULT_PROFILE_ID = "fast" DEFAULT_FRAME_SIZE_SECONDS = 3.0 @@ -40,10 +42,10 @@ def normalize_model_artifact_metadata( metadata: dict[str, object], ) -> dict[str, object]: """Validates and normalizes artifact metadata to the current envelope shape.""" - return normalize_v2_artifact_metadata( - metadata, - artifact_version=MODEL_ARTIFACT_VERSION, - ) + artifact_version = metadata.get("artifact_version") + if artifact_version == 2: + return normalize_v2_artifact_metadata(metadata, artifact_version=2) + return normalize_v3_artifact_metadata(metadata, artifact_version=MODEL_ARTIFACT_VERSION) def build_model_artifact_metadata( @@ -61,9 +63,16 @@ def build_model_artifact_metadata( torch_device: str | None = None, torch_dtype: str | None = None, provenance: dict[str, object] | None = None, + recipe_digest: str | None = None, + split_ledger_digest: str | None = None, + model_revision: str | None = None, + task_heads: list[str] | None = None, + sampling_policy: dict[str, object] | None = None, + seed: int | None = None, + evaluation_summary: dict[str, object] | None = None, ) -> dict[str, object]: """Builds normalized artifact metadata for persisted model envelopes.""" - return build_v2_artifact_metadata( + return build_v3_artifact_metadata( artifact_version=MODEL_ARTIFACT_VERSION, artifact_schema_version=ARTIFACT_SCHEMA_VERSION, feature_vector_size=feature_vector_size, @@ -79,6 +88,13 @@ def build_model_artifact_metadata( torch_device=torch_device, torch_dtype=torch_dtype, provenance=provenance, + recipe_digest=recipe_digest, + split_ledger_digest=split_ledger_digest, + model_revision=model_revision, + task_heads=task_heads or ["primary_emotion"], + sampling_policy=sampling_policy, + seed=seed, + evaluation_summary=evaluation_summary, ) @@ -97,6 +113,13 @@ def build_model_artifact( torch_device: str | None = None, torch_dtype: str | None = None, provenance: dict[str, object] | None = None, + recipe_digest: str | None = None, + split_ledger_digest: str | None = None, + model_revision: str | None = None, + task_heads: list[str] | None = None, + sampling_policy: dict[str, object] | None = None, + seed: int | None = None, + evaluation_summary: dict[str, object] | None = None, ) -> dict[str, object]: """Constructs one versioned model artifact envelope for safe persistence.""" metadata = build_model_artifact_metadata( @@ -113,6 +136,13 @@ def build_model_artifact( torch_device=torch_device, torch_dtype=torch_dtype, provenance=provenance, + recipe_digest=recipe_digest, + split_ledger_digest=split_ledger_digest, + model_revision=model_revision, + task_heads=task_heads, + sampling_policy=sampling_policy, + seed=seed, + evaluation_summary=evaluation_summary, ) return build_model_artifact_envelope( artifact_version=MODEL_ARTIFACT_VERSION, @@ -126,6 +156,7 @@ def deserialize_model_artifact(payload: object) -> LoadedModel: return deserialize_model_artifact_envelope( payload, artifact_version=MODEL_ARTIFACT_VERSION, + supported_artifact_versions=SUPPORTED_MODEL_ARTIFACT_VERSIONS, model_instance_check=lambda model: isinstance(model, MLPClassifier | Pipeline), normalize_metadata=normalize_model_artifact_metadata, read_positive_int=read_positive_int, diff --git a/ser/_internal/models/artifact_loading.py b/ser/_internal/models/artifact_loading.py index 173b7fc..a6d76bc 100644 --- a/ser/_internal/models/artifact_loading.py +++ b/ser/_internal/models/artifact_loading.py @@ -71,6 +71,7 @@ def deserialize_model_artifact_envelope( payload: object, *, artifact_version: int, + supported_artifact_versions: frozenset[int] | None = None, model_instance_check: Callable[[object], bool], normalize_metadata: Callable[[dict[str, object]], dict[str, object]], read_positive_int: Callable[[dict[str, object], str], int], @@ -87,10 +88,11 @@ def deserialize_model_artifact_envelope( ) resolved_artifact_version = payload.get("artifact_version") - if resolved_artifact_version != artifact_version: + supported_versions = supported_artifact_versions or frozenset({artifact_version}) + if resolved_artifact_version not in supported_versions: raise ValueError( "Unsupported model artifact version " - f"{resolved_artifact_version!r}; expected {artifact_version}. " + f"{resolved_artifact_version!r}; supported versions are {sorted(supported_versions)}. " "Regenerate artifacts with current training code." ) @@ -103,6 +105,8 @@ def deserialize_model_artifact_envelope( metadata_obj = payload.get("metadata") if not isinstance(metadata_obj, dict): raise ValueError("Model artifact metadata is missing or invalid.") + if metadata_obj.get("artifact_version") != resolved_artifact_version: + raise ValueError("Model artifact envelope and metadata versions must match.") normalized_metadata = normalize_metadata(metadata_obj) expected_feature_size = read_positive_int( normalized_metadata, diff --git a/ser/_internal/models/artifact_metadata.py b/ser/_internal/models/artifact_metadata.py index 408c2fc..4e0d36c 100644 --- a/ser/_internal/models/artifact_metadata.py +++ b/ser/_internal/models/artifact_metadata.py @@ -2,8 +2,12 @@ from __future__ import annotations +import json +import re from datetime import UTC, datetime +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + def read_positive_int(metadata: dict[str, object], field_name: str) -> int: """Returns a required positive integer field from metadata.""" @@ -193,3 +197,116 @@ def build_v2_artifact_metadata( payload, artifact_version=artifact_version, ) + + +def _read_optional_digest(metadata: dict[str, object], field_name: str) -> str | None: + raw_value = metadata.get(field_name) + if raw_value is None: + return None + if not isinstance(raw_value, str) or _SHA256_PATTERN.fullmatch(raw_value) is None: + raise ValueError(f"Model artifact metadata contains invalid {field_name!r} value.") + return raw_value + + +def _read_json_object(metadata: dict[str, object], field_name: str) -> dict[str, object]: + raw_value = metadata.get(field_name, {}) + if not isinstance(raw_value, dict) or any(not isinstance(key, str) for key in raw_value): + raise ValueError(f"Model artifact metadata contains invalid {field_name!r} value.") + try: + json.dumps(raw_value, allow_nan=False, sort_keys=True) + except (TypeError, ValueError) as err: + raise ValueError( + f"Model artifact metadata contains non-JSON {field_name!r} value." + ) from err + return raw_value + + +def normalize_v3_artifact_metadata( + metadata: dict[str, object], + *, + artifact_version: int, +) -> dict[str, object]: + """Validates research provenance fields while preserving the v2 inference contract.""" + normalized = normalize_v2_artifact_metadata(metadata, artifact_version=artifact_version) + for field_name in ("recipe_digest", "split_ledger_digest"): + digest = _read_optional_digest(metadata, field_name) + if digest is not None: + normalized[field_name] = digest + model_revision = metadata.get("model_revision") + if model_revision is not None: + if not isinstance(model_revision, str) or not model_revision.strip(): + raise ValueError("Model artifact metadata contains invalid 'model_revision' value.") + normalized["model_revision"] = model_revision.strip() + task_heads = metadata.get("task_heads", ["primary_emotion"]) + if not isinstance(task_heads, list) or not task_heads: + raise ValueError("Model artifact metadata contains invalid 'task_heads' value.") + normalized_heads: list[str] = [] + for task_head in task_heads: + if not isinstance(task_head, str) or not task_head.strip(): + raise ValueError("Model artifact metadata contains invalid 'task_heads' value.") + normalized_heads.append(task_head.strip()) + normalized["task_heads"] = sorted(set(normalized_heads)) + seed = metadata.get("seed") + if seed is not None: + if not isinstance(seed, int) or isinstance(seed, bool) or seed < 0: + raise ValueError("Model artifact metadata contains invalid 'seed' value.") + normalized["seed"] = seed + normalized["sampling_policy"] = _read_json_object(metadata, "sampling_policy") + normalized["evaluation_summary"] = _read_json_object(metadata, "evaluation_summary") + return normalized + + +def build_v3_artifact_metadata( + *, + artifact_version: int, + artifact_schema_version: str, + feature_vector_size: int, + training_samples: int, + labels: list[str], + backend_id: str, + profile: str, + feature_dim: int | None, + frame_size_seconds: float, + frame_stride_seconds: float, + pooling_strategy: str, + backend_model_id: str | None, + torch_device: str | None, + torch_dtype: str | None, + provenance: dict[str, object] | None, + recipe_digest: str | None, + split_ledger_digest: str | None, + model_revision: str | None, + task_heads: list[str], + sampling_policy: dict[str, object] | None, + seed: int | None, + evaluation_summary: dict[str, object] | None, +) -> dict[str, object]: + """Builds v3 metadata with reproducible dataset and training provenance.""" + payload = build_v2_artifact_metadata( + artifact_version=artifact_version, + artifact_schema_version=artifact_schema_version, + feature_vector_size=feature_vector_size, + training_samples=training_samples, + labels=labels, + backend_id=backend_id, + profile=profile, + feature_dim=feature_dim, + frame_size_seconds=frame_size_seconds, + frame_stride_seconds=frame_stride_seconds, + pooling_strategy=pooling_strategy, + backend_model_id=backend_model_id, + torch_device=torch_device, + torch_dtype=torch_dtype, + provenance=provenance, + ) + optional_fields: dict[str, object | None] = { + "recipe_digest": recipe_digest, + "split_ledger_digest": split_ledger_digest, + "model_revision": model_revision, + "seed": seed, + } + payload.update((key, value) for key, value in optional_fields.items() if value is not None) + payload["task_heads"] = task_heads + payload["sampling_policy"] = sampling_policy or {} + payload["evaluation_summary"] = evaluation_summary or {} + return normalize_v3_artifact_metadata(payload, artifact_version=artifact_version) diff --git a/tests/suites/unit/models/test_artifact_upgrade_path.py b/tests/suites/unit/models/test_artifact_upgrade_path.py index 2973855..6684e94 100644 --- a/tests/suites/unit/models/test_artifact_upgrade_path.py +++ b/tests/suites/unit/models/test_artifact_upgrade_path.py @@ -52,6 +52,67 @@ def test_deserialize_round_trips_v2_artifact_metadata() -> None: assert loaded.artifact_metadata["artifact_version"] == artifact_envelope.MODEL_ARTIFACT_VERSION assert loaded.artifact_metadata["artifact_schema_version"] == "v2" assert loaded.artifact_metadata["labels"] == ["happy", "sad"] + assert loaded.artifact_metadata["task_heads"] == ["primary_emotion"] + + +def test_deserialize_loads_previous_v2_envelope() -> None: + """The immediately previous artifact envelope remains loadable after v3 promotion.""" + artifact = artifact_envelope.build_model_artifact( + model=_build_classifier(), + feature_vector_size=193, + training_samples=32, + labels=["happy", "sad"], + ) + artifact["artifact_version"] = 2 + metadata = artifact["metadata"] + assert isinstance(metadata, dict) + metadata["artifact_version"] = 2 + + loaded = artifact_envelope.deserialize_model_artifact(artifact) + + assert loaded.artifact_metadata is not None + assert loaded.artifact_metadata["artifact_version"] == 2 + + +def test_v3_artifact_persists_research_recipe_and_evaluation_provenance() -> None: + """New envelopes retain reproducibility and masked-head declarations.""" + artifact = artifact_envelope.build_model_artifact( + model=_build_classifier(), + feature_vector_size=193, + training_samples=32, + labels=["happy", "sad"], + recipe_digest="a" * 64, + split_ledger_digest="b" * 64, + model_revision="encoder-r1", + task_heads=["primary_emotion", "vad"], + sampling_policy={"corpus": "sqrt", "class": "inverse_sqrt"}, + seed=17, + evaluation_summary={"macro_per_corpus_uar": 0.61}, + ) + + loaded = artifact_envelope.deserialize_model_artifact(artifact) + + assert loaded.artifact_metadata is not None + assert loaded.artifact_metadata["recipe_digest"] == "a" * 64 + assert loaded.artifact_metadata["split_ledger_digest"] == "b" * 64 + assert loaded.artifact_metadata["task_heads"] == ["primary_emotion", "vad"] + assert loaded.artifact_metadata["seed"] == 17 + + +def test_deserialize_rejects_mismatched_envelope_and_metadata_versions() -> None: + """Legacy selection is controlled by one consistent artifact version.""" + artifact = artifact_envelope.build_model_artifact( + model=_build_classifier(), + feature_vector_size=193, + training_samples=32, + labels=["happy", "sad"], + ) + metadata = artifact["metadata"] + assert isinstance(metadata, dict) + metadata["artifact_version"] = 2 + + with pytest.raises(ValueError, match="envelope and metadata versions must match"): + artifact_envelope.deserialize_model_artifact(artifact) def test_deserialize_round_trips_medium_v2_artifact_metadata() -> None: From 407553dae857a3f5a3c12739b0b37a7dc8a428b2 Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:18:49 -0300 Subject: [PATCH 06/10] feat(training): add readiness contract and fault containment Training could previously fail most of the way through a long run for reasons knowable up front: missing media, an infeasible split, no write permission, a full disk. Readiness checks those first and reports every problem at once rather than failing on the first. Failures carry a stable scope, reason, severity, and disposition. Broken checkouts, invalid configuration, leakage, insufficient class support, non-finite backend output, and resource shortages always abort. Only known sample-local decode failures or a missing path proven inside a registered root may be quarantined, and every proposed exclusion must satisfy the absolute, global, per-corpus, per-class, per-reason, and minimum-remaining budgets, so containment cannot silently delete a class or a corpus. Reports publish atomically; no partial document is ever treated as ready. --- ser/_internal/models/training_readiness.py | 2917 +++++++++++++++++ .../unit/models/test_training_readiness.py | 908 +++++ .../test_training_readiness_evidence.py | 430 +++ 3 files changed, 4255 insertions(+) create mode 100644 ser/_internal/models/training_readiness.py create mode 100644 tests/suites/unit/models/test_training_readiness.py create mode 100644 tests/suites/unit/models/test_training_readiness_evidence.py diff --git a/ser/_internal/models/training_readiness.py b/ser/_internal/models/training_readiness.py new file mode 100644 index 0000000..fb1727e --- /dev/null +++ b/ser/_internal/models/training_readiness.py @@ -0,0 +1,2917 @@ +"""Typed readiness, preparation-plan, and fault-containment contracts for training.""" + +from __future__ import annotations + +import errno +import hashlib +import importlib.util +import json +import logging +import math +import os +import resource +import shutil +import signal +import subprocess +import tempfile +import threading +import time +from bisect import bisect_left +from collections import Counter, defaultdict +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import asdict, dataclass, field, fields, is_dataclass, replace +from datetime import UTC, datetime +from enum import StrEnum +from pathlib import Path +from typing import Protocol, cast + +import numpy as np +import soundfile as sf + +from ser.config import AppConfig +from ser.profiles import ProfileName, resolve_profile_name + +from ..data.application.registry_snapshot import ( # noqa: TID251 + collect_dataset_registry_snapshot, +) +from ..data.manifest import Utterance # noqa: TID251 +from ..utils.audio_utils import AudioDecodeError, AudioIntegrityError # noqa: TID251 + +logger: logging.Logger = logging.getLogger(__name__) +READINESS_SCHEMA_VERSION = 1 +PREPARED_PLAN_SCHEMA_VERSION = 2 +PREPARATION_CODE_VERSION = "ser-training-preparation-v2" +PREPARED_CACHE_VERSION = "embedding-cache-v1" +PROFILE_CACHE_NAMESPACES: Mapping[ProfileName, str] = { + "fast": "fast_features", + "medium": "medium_embeddings", + "accurate": "accurate_embeddings", + "accurate-research": "accurate_research_embeddings", +} +DEFAULT_SMOKE_SAMPLE_CAP = 16 +_GIT_LFS_PREFIX = b"version https://git-lfs.github.com/spec/v1" +_TRANSIENT_LOCAL_IO_ERRNOS = frozenset({errno.EAGAIN, errno.EBUSY, errno.EINTR, errno.ETIMEDOUT}) + + +def _readiness_status(findings: Sequence[ReadinessFinding]) -> str: + """Returns a concise status label for one readiness phase.""" + if any(finding.blocking for finding in findings): + return "FAIL" + return "WARN" if findings else "PASS" + + +def _readiness_log_level(status: str) -> int: + """Returns a standard log level for one readiness status label.""" + if status == "FAIL": + return logging.ERROR + if status == "WARN": + return logging.WARNING + return logging.INFO + + +def _log_repair_record(record: RepairRecord) -> None: + """Emits one bounded repair action result.""" + level = logging.INFO if record.succeeded else logging.WARNING + logger.log( + level, + "REPAIR_ACTION action=%s target=%s succeeded=%s detail=%s", + record.action, + record.target[:200], + record.succeeded, + record.message[:500], + ) + + +def _progress_interval(total: int) -> int: + """Returns a low-volume progress interval for large readiness inventories.""" + return max(1, total // 10) + + +def _should_log_progress( + *, + processed: int, + total: int, + last_logged_at: float, + now: float, +) -> bool: + """Returns whether one bounded progress event should be emitted.""" + return ( + processed == 1 + or processed == total + or processed % _progress_interval(total) == 0 + or (now - last_logged_at) >= 30.0 + ) + + +class FailureScope(StrEnum): + """Scope at which a training failure is known to apply.""" + + RUN = "run" + CORPUS = "corpus" + SAMPLE = "sample" + WINDOW = "window" + CACHE = "cache" + OPTIONAL_ARTIFACT = "optional_artifact" + + +class FailureDisposition(StrEnum): + """Permitted action after one classified failure.""" + + ABORT = "abort" + REPAIR_THEN_RETRY = "repair_then_retry" + BOUNDED_RETRY = "bounded_retry" + RECOMPUTE = "recompute" + QUARANTINE = "quarantine" + CONTINUE = "continue" + + +class FailureSeverity(StrEnum): + """Structured severity for training findings.""" + + DEBUG = "debug" + INFO = "info" + WARNING = "warning" + ERROR = "error" + + +class FailureReasonCode(StrEnum): + """Stable reason codes emitted by readiness and containment services.""" + + INVALID_CONFIGURATION = "invalid_configuration" + DATASET_NOT_FOUND = "dataset_not_found" + REGISTRY_UNHEALTHY = "registry_unhealthy" + MANIFEST_INVALID = "manifest_invalid" + MEDIA_MISSING = "media_missing" + MEDIA_NOT_REGULAR = "media_not_regular" + MEDIA_EMPTY = "media_empty" + MEDIA_DECODE_FAILED = "media_decode_failed" + GIT_LFS_POINTER = "git_lfs_pointer" + DUPLICATE_SAMPLE_ID = "duplicate_sample_id" + DUPLICATE_CONTENT = "duplicate_content" + PATH_ALIAS = "path_alias" + INSUFFICIENT_CLASS_SUPPORT = "insufficient_class_support" + SPLIT_LEAKAGE = "split_leakage" + OUTPUT_UNWRITABLE = "output_unwritable" + DISK_SPACE_LOW = "disk_space_low" + RESOURCE_LIMIT = "resource_limit" + BACKEND_UNAVAILABLE = "backend_unavailable" + BACKEND_SMOKE_TIMEOUT = "backend_smoke_timeout" + BACKEND_OUTPUT_INVALID = "backend_output_invalid" + SAMPLE_AUDIO_CORRUPT = "sample_audio_corrupt" + SAMPLE_AUDIO_MISSING = "sample_audio_missing" + WINDOW_LOW_VARIANCE = "window_low_variance" + CACHE_CORRUPT = "cache_corrupt" + OPTIONAL_ARTIFACT_FAILED = "optional_artifact_failed" + QUARANTINE_BUDGET_EXCEEDED = "quarantine_budget_exceeded" + PREPARED_PLAN_INVALID = "prepared_plan_invalid" + REPAIR_FAILED = "repair_failed" + + +class TrainingMode(StrEnum): + """Training orchestration mode selected by the CLI or library boundary.""" + + TRAIN = "train" + DRY_RUN = "dry_run" + PREPARE_ONLY = "prepare_only" + + +@dataclass(frozen=True, slots=True) +class TrainingOperation: + """Mode and checkpoint inputs for one training invocation.""" + + mode: TrainingMode = TrainingMode.TRAIN + repair: bool = False + prepared_plan: Path | None = None + + def validate(self) -> None: + """Validates mutually compatible operation flags.""" + if self.repair and self.mode not in {TrainingMode.DRY_RUN, TrainingMode.PREPARE_ONLY}: + raise ValueError("--repair is valid only with --dry-run or --prepare-only.") + if self.prepared_plan is not None and self.mode is not TrainingMode.TRAIN: + raise ValueError("--prepared-plan is valid only for real training.") + + +@dataclass(frozen=True, slots=True) +class FailureClassification: + """Typed failure classification used before any continuation decision.""" + + scope: FailureScope + reason_code: FailureReasonCode + disposition: FailureDisposition + severity: FailureSeverity + diagnostic: str + + +@dataclass(frozen=True, slots=True) +class QuarantinePolicy: + """Bias-aware limits for excluding proven sample-local failures.""" + + max_absolute: int + max_global_ratio: float + max_corpus_ratio: float + max_class_ratio: float + max_per_reason: int + min_remaining_per_class_split: int + strict: bool = False + + def __post_init__(self) -> None: + """Validates all budget boundaries eagerly.""" + for name, value in ( + ("max_absolute", self.max_absolute), + ("max_per_reason", self.max_per_reason), + ("min_remaining_per_class_split", self.min_remaining_per_class_split), + ): + if value < 0: + raise ValueError(f"{name} must be non-negative.") + for name, ratio in ( + ("max_global_ratio", self.max_global_ratio), + ("max_corpus_ratio", self.max_corpus_ratio), + ("max_class_ratio", self.max_class_ratio), + ): + if not math.isfinite(ratio) or not 0.0 <= ratio <= 1.0: + raise ValueError(f"{name} must be finite and within [0, 1].") + + @classmethod + def from_settings(cls, settings: AppConfig) -> QuarantinePolicy: + """Builds the typed policy while preserving the legacy ratio surface.""" + config = settings.data_loader + return cls( + max_absolute=config.max_failed_files, + max_global_ratio=config.max_failed_file_ratio, + max_corpus_ratio=config.max_failed_file_ratio_per_corpus, + max_class_ratio=config.max_failed_file_ratio_per_class, + max_per_reason=config.max_failures_per_reason, + min_remaining_per_class_split=config.min_remaining_per_class_split, + strict=config.strict_quarantine, + ) + + +@dataclass(frozen=True, slots=True) +class QuarantineRecord: + """One deterministic, bounded quarantine-ledger record.""" + + sample_id: str + corpus: str + path_digest: str + primary_class: str + split: str + scope: FailureScope + reason_code: FailureReasonCode + diagnostic: str + first_occurrence: str + last_occurrence: str + retry_count: int + disposition: FailureDisposition = FailureDisposition.QUARANTINE + + def to_dict(self) -> dict[str, object]: + """Returns one canonical JSON-compatible ledger row.""" + return { + "sample_id": self.sample_id, + "corpus": self.corpus, + "path_digest": self.path_digest, + "primary_class": self.primary_class, + "split": self.split, + "scope": self.scope.value, + "reason_code": self.reason_code.value, + "diagnostic": self.diagnostic[:500], + "first_occurrence": self.first_occurrence, + "last_occurrence": self.last_occurrence, + "retry_count": self.retry_count, + "disposition": self.disposition.value, + } + + def identity_dict(self) -> dict[str, object]: + """Returns stable ledger identity without audit-only wall-clock fields.""" + payload = self.to_dict() + payload.pop("first_occurrence") + payload.pop("last_occurrence") + return payload + + +class QuarantineBudgetExceeded(RuntimeError): + """Raised when one projected exclusion violates any quarantine invariant.""" + + +class TrainingReadinessError(RuntimeError): + """Raised when mandatory readiness validation contains blocking findings.""" + + def __init__(self, report: ReadinessReport) -> None: + super().__init__("Training readiness validation failed; inspect the readiness report.") + self.report = report + + +class WindowContainmentError(ValueError): + """Known isolated window rejection that may be dropped with accounting.""" + + +class CacheEntryCorruptError(ValueError): + """Known derived-cache corruption that may be recomputed.""" + + +class OptionalArtifactError(OSError): + """Known failure of a declared optional artifact.""" + + +class PreparedPlanError(RuntimeError): + """Raised when a prepared plan is malformed, stale, or digest-invalid.""" + + +@dataclass(frozen=True, slots=True) +class ReadinessFinding: + """One stable readiness validation result.""" + + check: str + reason_code: FailureReasonCode + severity: FailureSeverity + message: str + blocking: bool + scope: FailureScope = FailureScope.RUN + sample_id: str | None = None + corpus: str | None = None + + def to_dict(self) -> dict[str, object]: + """Returns a JSON-compatible finding.""" + return { + "check": self.check, + "reason_code": self.reason_code.value, + "severity": self.severity.value, + "message": self.message, + "blocking": self.blocking, + "scope": self.scope.value, + "sample_id": self.sample_id, + "corpus": self.corpus, + } + + +@dataclass(frozen=True, slots=True) +class RepairRecord: + """Audit record for one explicitly requested allowlisted repair.""" + + action: str + target: str + succeeded: bool + message: str + + def to_dict(self) -> dict[str, object]: + """Returns a JSON-compatible repair record.""" + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class SmokeResult: + """Bounded backend-smoke output verified by the readiness service.""" + + attempted: int + succeeded: int + feature_dim: int + cache_round_trip: bool + backend_id: str + model_id: str + device: str + dtype: str + + +@dataclass(frozen=True, slots=True) +class ReadinessReport: + """Durable result of one complete readiness pass.""" + + schema_version: int + created_at: str + profile: ProfileName + settings_digest: str + registry_digest: str + manifest_digest: str + media_digest: str + selected_sample_ids: tuple[str, ...] + findings: tuple[ReadinessFinding, ...] + repairs: tuple[RepairRecord, ...] = () + smoke: SmokeResult | None = None + retries: int = 0 + recomputed_cache_entries: int = 0 + quarantined_samples: int = 0 + dropped_windows: int = 0 + quarantine: tuple[QuarantineRecord, ...] = () + effective_sample_ids: tuple[str, ...] = () + + @property + def ready(self) -> bool: + """Returns whether no blocking validation result remains.""" + return not any(finding.blocking for finding in self.findings) + + def to_dict(self) -> dict[str, object]: + """Returns the stable readiness-report payload.""" + return { + "schema_version": self.schema_version, + "created_at": self.created_at, + "profile": self.profile, + "ready": self.ready, + "digests": { + "settings": self.settings_digest, + "registry": self.registry_digest, + "manifest": self.manifest_digest, + "media": self.media_digest, + }, + "selected_sample_ids": list(self.selected_sample_ids), + "findings": [finding.to_dict() for finding in self.findings], + "repairs": [repair.to_dict() for repair in self.repairs], + "smoke": asdict(self.smoke) if self.smoke is not None else None, + "statistics": { + "retries": self.retries, + "recomputed_cache_entries": self.recomputed_cache_entries, + "quarantined_samples": self.quarantined_samples, + "dropped_windows": self.dropped_windows, + }, + "quarantine": [record.to_dict() for record in self.quarantine], + "effective_sample_ids": list(self.effective_sample_ids), + } + + +@dataclass(frozen=True, slots=True) +class PreparedPlan: + """Digest-protected declaration of reusable training preparation state.""" + + schema_version: int + created_at: str + code_version: str + package_version: str + profile: ProfileName + backend_id: str + model_id: str + model_revision: str + device: str + dtype: str + settings_digest: str + registry_digest: str + manifest_digest: str + media_digest: str + recipe_digest: str + split_ledger_digest: str + quarantine_ledger_digest: str + cache_namespace: str + cache_version: str + cache_keys: tuple[str, ...] + effective_counts: Mapping[str, Mapping[str, int]] + sample_ledger: tuple[Mapping[str, object], ...] + window_ledger: tuple[Mapping[str, object], ...] + disposition_counts: Mapping[str, int] + feature_shape: tuple[int, int] + feature_dtype: str + windowing_policy: Mapping[str, object] + noise_statistics: Mapping[str, object] + validation_findings: tuple[Mapping[str, object], ...] + repairs: tuple[Mapping[str, object], ...] + payload_path: str + payload_digest: str + overall_digest: str = field(default="") + + def unsigned_dict(self) -> dict[str, object]: + """Returns canonical plan content excluding its self-authenticating digest.""" + payload = asdict(self) + payload.pop("overall_digest", None) + return cast(dict[str, object], _json_value(payload)) + + def to_dict(self) -> dict[str, object]: + """Returns canonical plan content including the overall digest.""" + payload = self.unsigned_dict() + payload["overall_digest"] = self.overall_digest + return payload + + +class SmokeRunner(Protocol): + """Runs bounded real-backend validation for selected samples.""" + + def __call__( + self, + *, + settings: AppConfig, + samples: tuple[Utterance, ...], + probe_cache_dir: Path, + ) -> SmokeResult: ... + + +def _json_value(value: object) -> object: + """Normalizes dataclass/config values into deterministic JSON-compatible forms.""" + if is_dataclass(value) and not isinstance(value, type): + # Walk fields directly instead of asdict(): asdict deep-copies leaf values, + # which raises TypeError on read-only fields (e.g. a MappingProxyType). The + # per-field recursion is digest-equivalent because canonical_json_bytes sorts. + return {member.name: _json_value(getattr(value, member.name)) for member in fields(value)} + if isinstance(value, Path): + return str(value.expanduser().resolve(strict=False)) + if isinstance(value, StrEnum): + return value.value + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in sorted(value.items(), key=str)} + if isinstance(value, tuple | list): + return [_json_value(item) for item in value] + if isinstance(value, set | frozenset): + return sorted((_json_value(item) for item in value), key=repr) + if isinstance(value, float): + return value if math.isfinite(value) else f"nonfinite:{value!r}" + if value is None or isinstance(value, str | int | bool): + return value + return repr(value) + + +def canonical_json_bytes(payload: object) -> bytes: + """Serializes one normalized payload with stable ordering and separators.""" + return json.dumps( + _json_value(payload), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def digest_payload(payload: object) -> str: + """Returns the SHA-256 digest for one canonical payload.""" + return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + + +def hash_file(path: Path) -> str: + """Returns a streaming SHA-256 file digest.""" + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def normalized_pcm_digest(path: Path, *, sample_id: str | None = None) -> str: + """Streams a container-independent mono float32 PCM identity.""" + started_at = time.perf_counter() + digest = hashlib.sha256() + with sf.SoundFile(path) as handle: + digest.update(str(handle.samplerate).encode("ascii")) + frames_read = 0 + last_progress_at = started_at + for block in handle.blocks(blocksize=65_536, dtype="float32", always_2d=True): + mono = np.asarray(block, dtype="= 30.0: + logger.info( + "DATASET_MEDIA_HASH_PROGRESS sample=%s frames=%d total_frames=%d elapsed=%.1fs", + sample_id[:80], + frames_read, + int(handle.frames), + progress_now - started_at, + ) + last_progress_at = progress_now + return digest.hexdigest() + + +def expected_cache_namespace(settings: AppConfig) -> str: + """Returns the code-owned cache namespace for the selected profile.""" + return PROFILE_CACHE_NAMESPACES[resolve_profile_name(settings)] + + +def validated_cache_root(settings: AppConfig, namespace: str) -> Path: + """Resolves one relative app-owned cache namespace without path escape.""" + candidate = Path(namespace) + if candidate.is_absolute() or not namespace.strip() or ".." in candidate.parts: + raise PreparedPlanError("Prepared cache namespace must be a safe relative path.") + root = (settings.tmp_folder / candidate).resolve(strict=False) + tmp_root = settings.tmp_folder.resolve(strict=False) + if not root.is_relative_to(tmp_root): + raise PreparedPlanError("Prepared cache namespace escapes the application temp root.") + return root + + +def quarantine_ledger_digest(records: Iterable[QuarantineRecord]) -> str: + """Returns stable quarantine identity while retaining timestamps in audit rows.""" + rows = [ + record.identity_dict() + for record in sorted(records, key=lambda row: (row.sample_id, row.reason_code.value)) + ] + return digest_payload(rows) + + +def atomic_write_json(path: Path, payload: object) -> None: + """Atomically publishes one JSON document and removes staging files on failure.""" + path.parent.mkdir(parents=True, exist_ok=True) + encoded = ( + json.dumps( + _json_value(payload), + indent=2, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + ) + + "\n" + ) + descriptor, raw_tmp_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + tmp_path = Path(raw_tmp_path) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, path) + finally: + tmp_path.unlink(missing_ok=True) + + +def write_quarantine_ledger(path: Path, records: Iterable[QuarantineRecord]) -> str: + """Atomically writes audit JSONL and returns its stable identity digest.""" + sorted_records = sorted(records, key=lambda row: (row.sample_id, row.reason_code.value)) + rows = [record.to_dict() for record in sorted_records] + payload = b"".join(canonical_json_bytes(row) + b"\n" for row in rows) + path.parent.mkdir(parents=True, exist_ok=True) + descriptor, raw_tmp_path = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + tmp_path = Path(raw_tmp_path) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(payload) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, path) + finally: + tmp_path.unlink(missing_ok=True) + return quarantine_ledger_digest(sorted_records) + + +def select_smoke_samples( + utterances: Sequence[Utterance], + *, + cap: int = DEFAULT_SMOKE_SAMPLE_CAP, +) -> tuple[Utterance, ...]: + """Selects a deterministic bounded matrix covering corpus/format/read/language strata.""" + if cap <= 0: + raise ValueError("Smoke sample cap must be positive.") + + strata: dict[tuple[str, str, str, str], Utterance] = {} + coverage_keys: list[tuple[str, str, str, str, str, str, str]] = [] + coverage_candidates: dict[tuple[str, str, str, str, str, str, str], Utterance] = {} + for utterance in utterances: + suffix = utterance.audio_path.suffix.lower() or "" + read_kind = "segment" if utterance.start_seconds is not None else "full" + stratum = (utterance.corpus, suffix, read_kind, utterance.language or "") + representative = strata.get(stratum) + if representative is None: + if len(strata) == cap: + raise ValueError( + "Smoke sample cap cannot cover every required " + "corpus/format/read/language stratum: " + f"required>{cap} cap={cap}." + ) + strata[stratum] = utterance + elif (utterance.sample_id, str(utterance.audio_path)) < ( + representative.sample_id, + str(representative.audio_path), + ): + strata[stratum] = utterance + + coverage_key = ( + utterance.label or "", + utterance.language or "", + utterance.corpus, + utterance.sample_id, + str(utterance.audio_path), + str(utterance.start_seconds), + str(utterance.duration_seconds), + ) + insertion_index = bisect_left(coverage_keys, coverage_key) + if insertion_index == len(coverage_keys) or coverage_keys[insertion_index] != coverage_key: + coverage_keys.insert(insertion_index, coverage_key) + coverage_candidates[coverage_key] = utterance + if len(coverage_keys) > cap: + removed_key = coverage_keys.pop() + coverage_candidates.pop(removed_key) + + selected: list[Utterance] = [] + seen_ids: set[str] = set() + for key in sorted(strata): + candidate = strata[key] + selected.append(candidate) + seen_ids.add(candidate.sample_id) + if len(selected) == cap: + return tuple(selected) + for coverage_key in coverage_keys: + candidate = coverage_candidates[coverage_key] + if candidate.sample_id in seen_ids: + continue + selected.append(candidate) + seen_ids.add(candidate.sample_id) + if len(selected) == cap: + break + return tuple(selected) + + +def classify_failure( + error: Exception, + *, + scope: FailureScope, + sample: Utterance | None = None, + allowed_roots: Sequence[Path] = (), +) -> FailureClassification: + """Classifies only known exception types; unknown failures remain aborting.""" + diagnostic = (str(error).strip() or type(error).__name__)[:500] + if isinstance(error, AudioIntegrityError) and "Git LFS" in diagnostic: + return FailureClassification( + scope=FailureScope.CORPUS, + reason_code=FailureReasonCode.GIT_LFS_POINTER, + disposition=FailureDisposition.ABORT, + severity=FailureSeverity.ERROR, + diagnostic=diagnostic, + ) + if scope is FailureScope.WINDOW and isinstance(error, WindowContainmentError): + return FailureClassification( + scope=scope, + reason_code=FailureReasonCode.WINDOW_LOW_VARIANCE, + disposition=FailureDisposition.CONTINUE, + severity=FailureSeverity.WARNING, + diagnostic=diagnostic, + ) + if scope is FailureScope.CACHE and isinstance(error, CacheEntryCorruptError): + return FailureClassification( + scope=scope, + reason_code=FailureReasonCode.CACHE_CORRUPT, + disposition=FailureDisposition.RECOMPUTE, + severity=FailureSeverity.WARNING, + diagnostic=diagnostic, + ) + if scope is FailureScope.OPTIONAL_ARTIFACT and isinstance(error, OptionalArtifactError): + return FailureClassification( + scope=scope, + reason_code=FailureReasonCode.OPTIONAL_ARTIFACT_FAILED, + disposition=FailureDisposition.CONTINUE, + severity=FailureSeverity.WARNING, + diagnostic=diagnostic, + ) + if scope is FailureScope.SAMPLE and ( + isinstance(error, TimeoutError | InterruptedError) + or (isinstance(error, OSError) and error.errno in _TRANSIENT_LOCAL_IO_ERRNOS) + ): + return FailureClassification( + scope=scope, + reason_code=FailureReasonCode.MEDIA_DECODE_FAILED, + disposition=FailureDisposition.BOUNDED_RETRY, + severity=FailureSeverity.WARNING, + diagnostic=diagnostic, + ) + missing_sample_is_proven = False + if scope is FailureScope.SAMPLE and isinstance(error, FileNotFoundError) and sample is not None: + filename = error.filename + if isinstance(filename, str): + failed_path = Path(filename).expanduser().resolve(strict=False) + sample_path = sample.audio_path.expanduser().resolve(strict=False) + missing_sample_is_proven = failed_path == sample_path and any( + sample_path.is_relative_to(root.expanduser().resolve(strict=False)) + for root in allowed_roots + ) + if missing_sample_is_proven: + return FailureClassification( + scope=scope, + reason_code=FailureReasonCode.SAMPLE_AUDIO_MISSING, + disposition=FailureDisposition.QUARANTINE, + severity=FailureSeverity.WARNING, + diagnostic=diagnostic, + ) + if scope is FailureScope.SAMPLE and isinstance(error, AudioDecodeError): + return FailureClassification( + scope=scope, + reason_code=FailureReasonCode.SAMPLE_AUDIO_CORRUPT, + disposition=FailureDisposition.QUARANTINE, + severity=FailureSeverity.WARNING, + diagnostic=diagnostic, + ) + return FailureClassification( + scope=scope, + reason_code=FailureReasonCode.BACKEND_OUTPUT_INVALID, + disposition=FailureDisposition.ABORT, + severity=FailureSeverity.ERROR, + diagnostic=diagnostic, + ) + + +def enforce_quarantine_budget( + *, + policy: QuarantinePolicy, + all_samples: Sequence[Utterance], + existing_records: Sequence[QuarantineRecord], + candidate: Utterance, + classification: FailureClassification, +) -> None: + """Rejects one projected quarantine when any global or bias-aware budget fails.""" + if classification.scope is not FailureScope.SAMPLE: + raise QuarantineBudgetExceeded("Only sample-local failures may be quarantined.") + if classification.disposition is not FailureDisposition.QUARANTINE: + raise QuarantineBudgetExceeded("Failure classification does not permit quarantine.") + if policy.strict: + raise QuarantineBudgetExceeded("Strict quarantine policy disables sample exclusion.") + if not all_samples: + raise QuarantineBudgetExceeded("Cannot quarantine from an empty sample inventory.") + projected = len(existing_records) + 1 + if projected > policy.max_absolute: + raise QuarantineBudgetExceeded("Absolute quarantine budget exceeded.") + if projected / len(all_samples) > policy.max_global_ratio: + raise QuarantineBudgetExceeded("Global quarantine ratio exceeded.") + + corpus_total = sum(item.corpus == candidate.corpus for item in all_samples) + corpus_failed = sum(row.corpus == candidate.corpus for row in existing_records) + 1 + if corpus_total <= 0 or corpus_failed / corpus_total > policy.max_corpus_ratio: + raise QuarantineBudgetExceeded("Per-corpus quarantine ratio exceeded.") + + label = candidate.require_label() + class_total = sum(item.label == label for item in all_samples) + class_failed = sum(row.primary_class == label for row in existing_records) + 1 + if class_total <= 0 or class_failed / class_total > policy.max_class_ratio: + raise QuarantineBudgetExceeded("Per-class quarantine ratio exceeded.") + + same_reason = sum(row.reason_code is classification.reason_code for row in existing_records) + 1 + if same_reason > policy.max_per_reason: + raise QuarantineBudgetExceeded("Per-reason systematic-failure threshold exceeded.") + + split = str(candidate.split or "unspecified") + remaining = sum( + item.label == label + and str(item.split or "unspecified") == split + and item.sample_id != candidate.sample_id + and all(record.sample_id != item.sample_id for record in existing_records) + for item in all_samples + ) + if remaining < policy.min_remaining_per_class_split: + raise QuarantineBudgetExceeded("Minimum remaining class/split support would be violated.") + + +def build_quarantine_record( + *, + sample: Utterance, + classification: FailureClassification, + occurred_at: str, + retry_count: int, +) -> QuarantineRecord: + """Builds a bounded ledger row without exposing a raw source path.""" + return QuarantineRecord( + sample_id=sample.sample_id, + corpus=sample.corpus, + path_digest=hashlib.sha256(str(sample.audio_path).encode("utf-8")).hexdigest(), + primary_class=sample.require_label(), + split=str(sample.split or "unspecified"), + scope=classification.scope, + reason_code=classification.reason_code, + diagnostic=classification.diagnostic, + first_occurrence=occurred_at, + last_occurrence=occurred_at, + retry_count=retry_count, + disposition=classification.disposition, + ) + + +def settings_digest(settings: AppConfig) -> str: + """Returns the complete effective-settings digest relevant to training.""" + return digest_payload(settings) + + +def default_readiness_report_path(settings: AppConfig) -> Path: + """Returns the atomic readiness-report location for the active profile.""" + return settings.tmp_folder / f"training-readiness-{resolve_profile_name(settings)}.json" + + +def default_prepared_plan_path(settings: AppConfig) -> Path: + """Returns the prepared-plan location for the active profile.""" + return settings.tmp_folder / f"prepared-training-{resolve_profile_name(settings)}.json" + + +def default_prepared_payload_path(settings: AppConfig) -> Path: + """Returns the reusable feature-payload location for the active profile.""" + return settings.tmp_folder / f"prepared-training-{resolve_profile_name(settings)}.npz" + + +def _inventory_digests( + settings: AppConfig, + utterances: Sequence[Utterance], + *, + media_utterances: Sequence[Utterance] | None = None, +) -> tuple[str, str, str]: + started_at = time.perf_counter() + logger.info( + "INVENTORY_DIGEST_START utterances=%d media_utterances=%d", + len(utterances), + len(media_utterances if media_utterances is not None else utterances), + ) + registry = collect_dataset_registry_snapshot(settings=settings) + registry_payload = { + "entries": [asdict(entry) for entry in registry.entries], + "issues": [asdict(issue) for issue in registry.issues], + } + manifest_rows = [ + { + "sample_id": item.sample_id, + "corpus": item.corpus, + "label": item.label, + "speaker_id": item.speaker_id, + "session_id": item.session_id, + "language": item.language, + "split": item.split, + "start_seconds": item.start_seconds, + "duration_seconds": item.duration_seconds, + "normalized_audio_sha256": item.normalized_audio_sha256, + "dataset_revision": item.dataset_revision, + } + for item in sorted(utterances, key=lambda row: row.sample_id) + ] + media_rows: list[dict[str, object]] = [] + media_inventory = utterances if media_utterances is None else media_utterances + sorted_media_inventory = sorted(media_inventory, key=lambda row: row.sample_id) + total = len(sorted_media_inventory) + last_progress_at = time.perf_counter() + hashed_count = 0 + for processed, item in enumerate(sorted_media_inventory, start=1): + path = item.audio_path.expanduser() + if not path.is_file(): + media_rows.append( + { + "sample_id": item.sample_id, + "status": "missing_or_non_regular", + } + ) + continue + stat = path.stat() + file_hash = hash_file(path) + hashed_count += 1 + media_rows.append( + { + "sample_id": item.sample_id, + "size": stat.st_size, + "mtime_ns": stat.st_mtime_ns, + "sha256": file_hash, + } + ) + progress_now = time.perf_counter() + if _should_log_progress( + processed=processed, + total=total, + last_logged_at=last_progress_at, + now=progress_now, + ): + logger.info( + "INVENTORY_DIGEST_PROGRESS checked=%d total=%d hashed=%d elapsed=%.1fs sample=%s", + processed, + total, + hashed_count, + progress_now - started_at, + item.sample_id[:80], + ) + last_progress_at = progress_now + result = ( + digest_payload(registry_payload), + digest_payload(manifest_rows), + digest_payload(media_rows), + ) + logger.info( + "INVENTORY_DIGEST_DONE media_rows=%d hashed=%d elapsed=%.1fs", + len(media_rows), + hashed_count, + time.perf_counter() - started_at, + ) + return result + + +def _configuration_findings(settings: AppConfig) -> list[ReadinessFinding]: + findings: list[ReadinessFinding] = [] + numeric_checks: tuple[tuple[str, float, Callable[[float], bool]], ...] = ( + ("training.test_size", settings.training.test_size, lambda value: 0.0 < value < 1.0), + ("training.dev_size", settings.training.dev_size, lambda value: 0.0 < value < 1.0), + ( + "data_loader.max_failed_file_ratio", + settings.data_loader.max_failed_file_ratio, + lambda value: 0.0 <= value <= 1.0, + ), + ( + "audio_read.retry_delay_seconds", + settings.audio_read.retry_delay_seconds, + lambda value: value >= 0.0, + ), + ( + "data_loader.max_failed_file_ratio_per_corpus", + settings.data_loader.max_failed_file_ratio_per_corpus, + lambda value: 0.0 <= value <= 1.0, + ), + ( + "data_loader.max_failed_file_ratio_per_class", + settings.data_loader.max_failed_file_ratio_per_class, + lambda value: 0.0 <= value <= 1.0, + ), + ("nn.alpha", settings.nn.alpha, lambda value: value >= 0.0), + ("nn.epsilon", settings.nn.epsilon, lambda value: value > 0.0), + ( + "medium_training.min_window_std", + settings.medium_training.min_window_std, + lambda value: value >= 0.0, + ), + ) + for name, value, predicate in numeric_checks: + if not math.isfinite(value) or not predicate(value): + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message=f"Invalid training setting {name}={value!r}.", + blocking=True, + ) + ) + if settings.training.test_size + settings.training.dev_size >= 1.0: + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message="training.test_size + training.dev_size must be below 1.0.", + blocking=True, + ) + ) + for profile_name, runtime in ( + ("fast", settings.fast_runtime), + ("medium", settings.medium_runtime), + ("accurate", settings.accurate_runtime), + ("accurate-research", settings.accurate_research_runtime), + ): + runtime_values = ( + ("timeout_seconds", runtime.timeout_seconds, lambda value: value >= 0.0), + ("retry_backoff_seconds", runtime.retry_backoff_seconds, lambda value: value >= 0.0), + ( + "pool_window_size_seconds", + runtime.pool_window_size_seconds, + lambda value: value > 0.0, + ), + ( + "pool_window_stride_seconds", + runtime.pool_window_stride_seconds, + lambda value: value > 0.0, + ), + ( + "post_hysteresis_enter_confidence", + runtime.post_hysteresis_enter_confidence, + lambda value: 0.0 <= value <= 1.0, + ), + ( + "post_hysteresis_exit_confidence", + runtime.post_hysteresis_exit_confidence, + lambda value: 0.0 <= value <= 1.0, + ), + ( + "post_min_segment_duration_seconds", + runtime.post_min_segment_duration_seconds, + lambda value: value >= 0.0, + ), + ) + for name, value, predicate in runtime_values: + if not math.isfinite(value) or not predicate(value): + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message=f"Invalid {profile_name} runtime setting {name}={value!r}.", + blocking=True, + ) + ) + if runtime.pool_window_stride_seconds > runtime.pool_window_size_seconds: + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message=f"{profile_name} pooling stride cannot exceed its window size.", + blocking=True, + ) + ) + if ( + runtime.max_timeout_retries < 0 + or runtime.max_transient_retries < 0 + or runtime.post_smoothing_window_frames <= 0 + ): + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message=f"{profile_name} retry/smoothing counts are invalid.", + blocking=True, + ) + ) + if ( + settings.training.random_state < 0 + or settings.nn.random_state < 0 + or settings.audio_read.max_retries < 0 + or settings.data_loader.max_workers <= 0 + or settings.data_loader.max_failed_files < 0 + or settings.data_loader.max_failures_per_reason < 0 + or settings.data_loader.min_remaining_per_class_split < 0 + or settings.nn.max_iter <= 0 + or settings.medium_training.max_windows_per_clip < 0 + ): + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message="Training seeds, retries, workers, iterations, and budgets are invalid.", + blocking=True, + ) + ) + requested_device = settings.torch_runtime.device.split(":", 1)[0] + if requested_device not in {"auto", "cpu", "cuda", "mps", "xpu"} or ( + settings.torch_runtime.dtype not in {"auto", "float16", "float32", "bfloat16"} + ): + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message="Torch device/dtype selector is unsupported.", + blocking=True, + ) + ) + if requested_device in {"cuda", "mps", "xpu"} and importlib.util.find_spec("torch") is None: + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.BACKEND_UNAVAILABLE, + severity=FailureSeverity.ERROR, + message=f"Requested device {requested_device!r} requires torch before backend load.", + blocking=True, + ) + ) + if ( + settings.nn.max_iter <= 0 + or settings.nn.alpha < 0.0 + or settings.nn.epsilon <= 0.0 + or any(size <= 0 for size in settings.nn.hidden_layer_sizes) + ): + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message="Classifier iteration, regularization, epsilon, and layer sizes are invalid.", + blocking=True, + ) + ) + if ( + settings.medium_training.min_window_std < 0.0 + or settings.medium_training.max_windows_per_clip < 0 + ): + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message="Medium noise-control thresholds must be non-negative.", + blocking=True, + ) + ) + if settings.audio_read.max_retries <= 0 or settings.data_loader.max_workers <= 0: + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message="Audio retry and dataset worker counts must be positive.", + blocking=True, + ) + ) + try: + QuarantinePolicy.from_settings(settings) + except ValueError as error: + findings.append( + ReadinessFinding( + check="configuration", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message=str(error), + blocking=True, + ) + ) + return findings + + +def _restricted_backend_access_findings(settings: AppConfig) -> list[ReadinessFinding]: + """Validates restricted backend policy before any backend construction.""" + if resolve_profile_name(settings) != "accurate-research": + return [] + from ser._internal.license_check import ( # noqa: TID251 + BackendLicensePolicyError, + ensure_backend_access, + load_persisted_backend_consents, + parse_allowed_restricted_backends_env, + ) + + try: + ensure_backend_access( + backend_id="emotion2vec", + restricted_backends_enabled=settings.runtime_flags.restricted_backends, + allowed_restricted_backends=parse_allowed_restricted_backends_env(), + persisted_consents=load_persisted_backend_consents(settings=settings), + ) + except (BackendLicensePolicyError, OSError, ValueError) as error: + return [ + ReadinessFinding( + check="restricted_backend_access", + reason_code=FailureReasonCode.BACKEND_UNAVAILABLE, + severity=FailureSeverity.ERROR, + message=str(error), + blocking=True, + ) + ] + return [] + + +def _registry_findings(settings: AppConfig) -> list[ReadinessFinding]: + snapshot = collect_dataset_registry_snapshot(settings=settings) + return [ + ReadinessFinding( + check="dataset_registry", + reason_code=( + FailureReasonCode.GIT_LFS_POINTER + if "lfs" in issue.code.lower() or "lfs" in issue.message.lower() + else FailureReasonCode.REGISTRY_UNHEALTHY + ), + severity=FailureSeverity.ERROR, + message=f"{issue.dataset_id}: {issue.message}", + blocking=True, + scope=FailureScope.CORPUS, + ) + for issue in snapshot.issues + ] + + +def _allowed_media_roots(settings: AppConfig) -> tuple[Path, ...]: + """Returns canonical roots that manifest media paths are permitted to inhabit.""" + snapshot = collect_dataset_registry_snapshot(settings=settings) + if snapshot.entries: + return tuple( + sorted({entry.dataset_root.expanduser() for entry in snapshot.entries}, key=str) + ) + roots = {settings.dataset.folder.expanduser()} + roots.update(path.expanduser().parent for path in settings.dataset.manifest_paths) + return tuple(sorted(roots, key=str)) + + +def _contain_readiness_sample_failures( + *, + settings: AppConfig, + utterances: Sequence[Utterance], + findings: Sequence[ReadinessFinding], + occurred_at: str, + allowed_roots: Sequence[Path] | None = None, +) -> tuple[list[ReadinessFinding], tuple[QuarantineRecord, ...], tuple[Utterance, ...]]: + """Converts allowlisted isolated media findings into budgeted durable exclusions.""" + started_at = time.perf_counter() + total_findings = len(findings) + last_progress_at = started_at + budget_exceeded = 0 + logger.info( + "DATASET_MEDIA_CONTAINMENT_START findings=%d samples=%d", + total_findings, + len(utterances), + ) + by_id = {item.sample_id: item for item in utterances} + policy = QuarantinePolicy.from_settings(settings) + resolved_allowed_roots = ( + tuple(allowed_roots) if allowed_roots is not None else _allowed_media_roots(settings) + ) + records: list[QuarantineRecord] = [] + resolved_findings: list[ReadinessFinding] = [] + local_reasons = { + FailureReasonCode.MEDIA_MISSING, + FailureReasonCode.MEDIA_EMPTY, + FailureReasonCode.MEDIA_DECODE_FAILED, + } + + def _maybe_log_containment_progress(processed: int) -> None: + nonlocal last_progress_at + progress_now = time.perf_counter() + if _should_log_progress( + processed=processed, + total=total_findings, + last_logged_at=last_progress_at, + now=progress_now, + ): + logger.info( + "DATASET_MEDIA_CONTAINMENT_PROGRESS processed=%d total=%d quarantined=%d " + "budget_exceeded=%d elapsed=%.1fs", + processed, + total_findings, + len(records), + budget_exceeded, + progress_now - started_at, + ) + last_progress_at = progress_now + + for processed, finding in enumerate(findings, start=1): + if ( + finding.scope is not FailureScope.SAMPLE + or finding.reason_code not in local_reasons + or finding.sample_id is None + ): + resolved_findings.append(finding) + _maybe_log_containment_progress(processed) + continue + sample = by_id.get(finding.sample_id) + if sample is None: + resolved_findings.append(finding) + _maybe_log_containment_progress(processed) + continue + error: Exception = ( + FileNotFoundError(errno.ENOENT, finding.message, str(sample.audio_path)) + if finding.reason_code is FailureReasonCode.MEDIA_MISSING + else AudioDecodeError(finding.message) + ) + classification = classify_failure( + error, + scope=FailureScope.SAMPLE, + sample=sample, + allowed_roots=resolved_allowed_roots, + ) + try: + enforce_quarantine_budget( + policy=policy, + all_samples=utterances, + existing_records=records, + candidate=sample, + classification=classification, + ) + except QuarantineBudgetExceeded as error: + budget_exceeded += 1 + resolved_findings.append(finding) + resolved_findings.append( + ReadinessFinding( + check="quarantine_budget", + reason_code=FailureReasonCode.QUARANTINE_BUDGET_EXCEEDED, + severity=FailureSeverity.ERROR, + message=f"{sample.sample_id}: {error}", + blocking=True, + scope=FailureScope.RUN, + sample_id=sample.sample_id, + corpus=sample.corpus, + ) + ) + _maybe_log_containment_progress(processed) + continue + record = build_quarantine_record( + sample=sample, + classification=classification, + occurred_at=occurred_at, + retry_count=0, + ) + records.append(record) + resolved_findings.append( + replace( + finding, + severity=FailureSeverity.WARNING, + message=f"{finding.message} Quarantined within all configured budgets.", + blocking=False, + ) + ) + _maybe_log_containment_progress(processed) + quarantined_ids = {record.sample_id for record in records} + effective = tuple(item for item in utterances if item.sample_id not in quarantined_ids) + logger.info( + "DATASET_MEDIA_CONTAINMENT_DONE findings=%d quarantined=%d effective=%d " + "budget_exceeded=%d elapsed=%.1fs", + total_findings, + len(records), + len(effective), + budget_exceeded, + time.perf_counter() - started_at, + ) + return resolved_findings, tuple(records), effective + + +def _media_findings( + utterances: Sequence[Utterance], + *, + allowed_roots: Sequence[Path], +) -> list[ReadinessFinding]: + findings: list[ReadinessFinding] = [] + seen_ids: set[str] = set() + seen_paths: dict[tuple[Path, float | None, float | None], str] = {} + seen_hashes: dict[tuple[str, float | None, float | None], str] = {} + started_at = time.perf_counter() + sorted_utterances = sorted(utterances, key=lambda row: row.sample_id) + total = len(sorted_utterances) + hashed_count = 0 + last_progress_at = started_at + logger.info("DATASET_MEDIA_CHECK_START samples=%d", total) + + def _maybe_log_media_progress(processed: int, item: Utterance, *, stage: str) -> None: + nonlocal last_progress_at + progress_now = time.perf_counter() + if _should_log_progress( + processed=processed, + total=total, + last_logged_at=last_progress_at, + now=progress_now, + ): + logger.info( + "DATASET_MEDIA_PROGRESS checked=%d total=%d hashed=%d findings=%d " + "elapsed=%.1fs sample=%s corpus=%s stage=%s", + processed, + total, + hashed_count, + len(findings), + progress_now - started_at, + item.sample_id[:80], + item.corpus[:80], + stage, + ) + last_progress_at = progress_now + + for processed, item in enumerate(sorted_utterances, start=1): + if item.sample_id in seen_ids: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.DUPLICATE_SAMPLE_ID, + severity=FailureSeverity.ERROR, + message=f"Duplicate sample ID {item.sample_id!r}.", + blocking=True, + scope=FailureScope.CORPUS, + ) + ) + seen_ids.add(item.sample_id) + path = item.audio_path.expanduser() + resolved_candidate = path.resolve(strict=False) + if not any( + resolved_candidate.is_relative_to(root.resolve(strict=False)) for root in allowed_roots + ): + findings.append( + ReadinessFinding( + check="manifest_integrity", + reason_code=FailureReasonCode.MANIFEST_INVALID, + severity=FailureSeverity.ERROR, + message=f"Media path escapes every allowed dataset root for {item.sample_id!r}.", + blocking=True, + scope=FailureScope.CORPUS, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + _maybe_log_media_progress(processed, item, stage="allowed_roots") + continue + if not path.exists(): + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.MEDIA_MISSING, + severity=FailureSeverity.ERROR, + message=f"Media is missing for sample {item.sample_id!r}.", + blocking=True, + scope=FailureScope.SAMPLE, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + _maybe_log_media_progress(processed, item, stage="exists") + continue + if not path.is_file(): + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.MEDIA_NOT_REGULAR, + severity=FailureSeverity.ERROR, + message=f"Media is not a regular file for sample {item.sample_id!r}.", + blocking=True, + scope=FailureScope.SAMPLE, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + _maybe_log_media_progress(processed, item, stage="is_file") + continue + file_size = path.stat().st_size + if file_size == 0: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.MEDIA_EMPTY, + severity=FailureSeverity.ERROR, + message=f"Media is empty for sample {item.sample_id!r}.", + blocking=True, + scope=FailureScope.SAMPLE, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + _maybe_log_media_progress(processed, item, stage="size") + continue + with path.open("rb") as handle: + if handle.read(len(_GIT_LFS_PREFIX)) == _GIT_LFS_PREFIX: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.GIT_LFS_POINTER, + severity=FailureSeverity.ERROR, + message=f"Git LFS pointer found in corpus {item.corpus!r}.", + blocking=True, + scope=FailureScope.CORPUS, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + _maybe_log_media_progress(processed, item, stage="lfs_pointer") + continue + resolved_path = path.resolve() + media_identity = (resolved_path, item.start_seconds, item.duration_seconds) + if media_identity in seen_paths: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.PATH_ALIAS, + severity=FailureSeverity.ERROR, + message=( + f"Samples {seen_paths[media_identity]!r} and {item.sample_id!r} " + "resolve to the same media path." + ), + blocking=True, + scope=FailureScope.CORPUS, + ) + ) + else: + seen_paths[media_identity] = item.sample_id + try: + info = sf.info(str(path)) + except PermissionError as error: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.MEDIA_DECODE_FAILED, + severity=FailureSeverity.ERROR, + message=f"Media permission failure for {item.sample_id!r}: {error}", + blocking=True, + scope=FailureScope.RUN, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + _maybe_log_media_progress(processed, item, stage="metadata") + continue + except (OSError, RuntimeError) as error: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.MEDIA_DECODE_FAILED, + severity=FailureSeverity.ERROR, + message=f"Media metadata decode failed for {item.sample_id!r}: {error}", + blocking=True, + scope=FailureScope.SAMPLE, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + _maybe_log_media_progress(processed, item, stage="metadata") + continue + if info.samplerate <= 0 or info.channels <= 0 or info.frames <= 0: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.MEDIA_DECODE_FAILED, + severity=FailureSeverity.ERROR, + message=f"Invalid audio metadata for sample {item.sample_id!r}.", + blocking=True, + scope=FailureScope.SAMPLE, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + media_duration = float(info.frames) / float(info.samplerate) + segment_end = float(item.start_seconds or 0.0) + float( + item.duration_seconds or media_duration + ) + if segment_end > media_duration + 1e-3: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.MEDIA_DECODE_FAILED, + severity=FailureSeverity.ERROR, + message=f"Segment bounds exceed media duration for {item.sample_id!r}.", + blocking=True, + scope=FailureScope.SAMPLE, + sample_id=item.sample_id, + corpus=item.corpus, + ) + ) + if item.normalized_audio_sha256: + content_hash = item.normalized_audio_sha256 + else: + if file_size >= 25 * 1024 * 1024: + logger.info( + "DATASET_MEDIA_HASH_START sample=%s size=%d path=%s", + item.sample_id[:80], + file_size, + path, + ) + hash_started_at = time.perf_counter() + content_hash = normalized_pcm_digest(path, sample_id=item.sample_id) + hashed_count += 1 + if file_size >= 25 * 1024 * 1024: + logger.info( + "DATASET_MEDIA_HASH_DONE sample=%s size=%d elapsed=%.1fs", + item.sample_id[:80], + file_size, + time.perf_counter() - hash_started_at, + ) + content_identity = (content_hash, item.start_seconds, item.duration_seconds) + if content_identity in seen_hashes: + findings.append( + ReadinessFinding( + check="dataset_media", + reason_code=FailureReasonCode.DUPLICATE_CONTENT, + severity=FailureSeverity.ERROR, + message=( + f"Samples {seen_hashes[content_identity]!r} and {item.sample_id!r} " + "have duplicate normalized content." + ), + blocking=True, + scope=FailureScope.CORPUS, + ) + ) + else: + seen_hashes[content_identity] = item.sample_id + _maybe_log_media_progress(processed, item, stage="hash") + logger.info( + "DATASET_MEDIA_CHECK_DONE samples=%d hashed=%d findings=%d elapsed=%.1fs", + total, + hashed_count, + len(findings), + time.perf_counter() - started_at, + ) + return findings + + +def _split_findings( + utterances: Sequence[Utterance], + *, + settings: AppConfig, +) -> list[ReadinessFinding]: + findings: list[ReadinessFinding] = [] + labels = Counter(item.label for item in utterances if item.label is not None) + if len(labels) < 2 or any(count < 2 for count in labels.values()): + findings.append( + ReadinessFinding( + check="split_feasibility", + reason_code=FailureReasonCode.INSUFFICIENT_CLASS_SUPPORT, + severity=FailureSeverity.ERROR, + message="At least two classes with two or more utterances each are required.", + blocking=True, + ) + ) + else: + from ser._internal.models.dataset_splitting import ( # noqa: TID251 + split_utterances_three_way, + ) + + try: + train_samples, dev_samples, test_samples, split_metadata = split_utterances_three_way( + samples=list(utterances), + settings=settings, + logger=logging.getLogger(__name__), + ) + minimum = settings.data_loader.min_remaining_per_class_split + for partition_name, partition in ( + ("train", train_samples), + ("dev", dev_samples), + ("test", test_samples), + ): + support = { + label: sum(item.label == label for item in partition) for label in labels + } + if any(count < minimum for count in support.values()): + findings.append( + ReadinessFinding( + check="split_feasibility", + reason_code=FailureReasonCode.INSUFFICIENT_CLASS_SUPPORT, + severity=FailureSeverity.ERROR, + message=( + f"{partition_name} split violates minimum class support: {support}." + ), + blocking=True, + ) + ) + if split_metadata.speaker_overlap_count > 0: + findings.append( + ReadinessFinding( + check="split_feasibility", + reason_code=FailureReasonCode.SPLIT_LEAKAGE, + severity=FailureSeverity.ERROR, + message="Deterministic split contains overlapping speakers.", + blocking=True, + ) + ) + except (RuntimeError, ValueError) as error: + findings.append( + ReadinessFinding( + check="split_feasibility", + reason_code=FailureReasonCode.INSUFFICIENT_CLASS_SUPPORT, + severity=FailureSeverity.ERROR, + message=str(error), + blocking=True, + ) + ) + for identity_name in ("speaker_id", "session_id", "normalized_audio_sha256"): + partition_by_identity: dict[str, set[str]] = defaultdict(set) + for item in utterances: + identity = getattr(item, identity_name) + if identity is not None and item.split is not None: + partition_by_identity[str(identity)].add(str(item.split)) + leaked = sorted( + key for key, partitions in partition_by_identity.items() if len(partitions) > 1 + ) + if leaked: + findings.append( + ReadinessFinding( + check="split_feasibility", + reason_code=FailureReasonCode.SPLIT_LEAKAGE, + severity=FailureSeverity.ERROR, + message=f"{identity_name} leakage across protected partitions ({len(leaked)}).", + blocking=True, + ) + ) + return findings + + +def _probe_directory(path: Path) -> None: + """Checks write and atomic-rename behavior without retaining probe artifacts.""" + existing_parent = path.expanduser() + while not existing_parent.exists() and existing_parent != existing_parent.parent: + existing_parent = existing_parent.parent + if not existing_parent.is_dir() or not os.access(existing_parent, os.W_OK): + raise OSError(f"Required path parent is not writable: {path}") + probe_dir = path if path.is_dir() else existing_parent + descriptor, raw_source = tempfile.mkstemp(prefix=".ser-write-probe-", dir=probe_dir) + source = Path(raw_source) + destination = source.with_suffix(".renamed") + try: + os.write(descriptor, b"ser") + os.fsync(descriptor) + os.close(descriptor) + descriptor = -1 + os.replace(source, destination) + finally: + if descriptor >= 0: + os.close(descriptor) + source.unlink(missing_ok=True) + destination.unlink(missing_ok=True) + + +def _resource_findings(settings: AppConfig, sample_count: int) -> list[ReadinessFinding]: + findings: list[ReadinessFinding] = [] + required_paths = { + settings.models.folder, + settings.tmp_folder, + settings.models.training_report_file.parent, + settings.models.model_cache_dir, + } + for path in sorted(required_paths, key=str): + try: + _probe_directory(path) + except OSError as error: + findings.append( + ReadinessFinding( + check="filesystem_resources", + reason_code=FailureReasonCode.OUTPUT_UNWRITABLE, + severity=FailureSeverity.ERROR, + message=str(error), + blocking=True, + ) + ) + existing_tmp_parent = settings.tmp_folder + while not existing_tmp_parent.exists() and existing_tmp_parent != existing_tmp_parent.parent: + existing_tmp_parent = existing_tmp_parent.parent + free_bytes = shutil.disk_usage(existing_tmp_parent).free + estimated_bytes = max(128 * 1024 * 1024, sample_count * 4 * 1024 * 1024) + if free_bytes < estimated_bytes * 2: + findings.append( + ReadinessFinding( + check="filesystem_resources", + reason_code=FailureReasonCode.DISK_SPACE_LOW, + severity=FailureSeverity.ERROR, + message=( + f"Available disk ({free_bytes} bytes) is below conservative requirement " + f"({estimated_bytes * 2} bytes)." + ), + blocking=True, + ) + ) + try: + soft_fd_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + except (OSError, ValueError): + soft_fd_limit = resource.RLIM_INFINITY + required_fds = max(64, settings.data_loader.max_workers * 4) + if soft_fd_limit != resource.RLIM_INFINITY and soft_fd_limit < required_fds: + findings.append( + ReadinessFinding( + check="filesystem_resources", + reason_code=FailureReasonCode.RESOURCE_LIMIT, + severity=FailureSeverity.ERROR, + message=f"File descriptor limit {soft_fd_limit} is below required {required_fds}.", + blocking=True, + ) + ) + try: + available_memory = int(os.sysconf("SC_AVPHYS_PAGES")) * int(os.sysconf("SC_PAGE_SIZE")) + except (OSError, ValueError, TypeError): + available_memory = 0 + required_memory = max(512 * 1024 * 1024, sample_count * 2 * 1024 * 1024) + if available_memory and available_memory < required_memory: + findings.append( + ReadinessFinding( + check="filesystem_resources", + reason_code=FailureReasonCode.RESOURCE_LIMIT, + severity=FailureSeverity.ERROR, + message=( + f"Available memory {available_memory} is below conservative requirement " + f"{required_memory}." + ), + blocking=True, + ) + ) + return findings + + +def _backend_failure_reason(error: Exception) -> FailureReasonCode: + """Maps known load/dependency failures without misclassifying output-contract defects.""" + if isinstance(error, TimeoutError): + return FailureReasonCode.BACKEND_SMOKE_TIMEOUT + if isinstance(error, ImportError): + return FailureReasonCode.BACKEND_UNAVAILABLE + message = str(error).lower() + if isinstance(error, RuntimeError) and any( + marker in message + for marker in ("dependency", "unavailable", "not installed", "failed to load", "model load") + ): + return FailureReasonCode.BACKEND_UNAVAILABLE + return FailureReasonCode.BACKEND_OUTPUT_INVALID + + +def _network_repairs_allowed() -> bool: + """Returns whether explicit policy permits repair-time network access.""" + return os.getenv("SER_TRAINING_REPAIR_ALLOW_NETWORK", "").strip().lower() in { + "1", + "true", + "yes", + "on", + } + + +def _run_repair_command( + command: Sequence[str], *, timeout_seconds: float = 120.0 +) -> tuple[bool, str]: + """Runs one checked repair command with timeout and bounded diagnostics.""" + try: + completed = subprocess.run( + list(command), + check=False, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) + except (OSError, subprocess.TimeoutExpired) as error: + return False, str(error)[:2000] + detail = (completed.stderr or completed.stdout or "completed").strip()[:2000] + return completed.returncode == 0, detail + + +def _git_root(path: Path) -> Path | None: + """Returns the nearest compatible Git checkout root.""" + candidate = path.expanduser().resolve(strict=False) + if candidate.is_file(): + candidate = candidate.parent + for parent in (candidate, *candidate.parents): + if (parent / ".git").exists(): + return parent + return None + + +def _apply_pre_validation_repairs(settings: AppConfig) -> list[RepairRecord]: + """Applies every safe local repair before running the same validation pass.""" + started_at = time.perf_counter() + logger.info("REPAIR_START") + records: list[RepairRecord] = [] + + def _record(record: RepairRecord) -> None: + records.append(record) + _log_repair_record(record) + + owned_directories = { + settings.tmp_folder, + settings.models.folder, + settings.models.model_cache_dir, + settings.models.training_report_file.parent, + } + for path in sorted(owned_directories, key=str): + try: + path.mkdir(parents=True, exist_ok=True) + except OSError as error: + _record(RepairRecord("create_application_directory", str(path), False, str(error))) + else: + _record( + RepairRecord( + "create_application_directory", + str(path), + True, + "Directory is present.", + ) + ) + + staging_patterns = (".ser-write-probe-*", ".prepared-training-*.*", ".training-readiness-*.*") + removed_staging = 0 + staging_failures = 0 + if settings.tmp_folder.is_dir(): + for pattern in staging_patterns: + for path in settings.tmp_folder.rglob(pattern): + if path.is_file(): + try: + path.unlink(missing_ok=True) + except OSError: + staging_failures += 1 + else: + removed_staging += 1 + _record( + RepairRecord( + "clean_application_staging", + str(settings.tmp_folder), + staging_failures == 0, + ( + f"Removed {removed_staging} abandoned staging/probe file(s); " + f"failures={staging_failures}." + ), + ) + ) + + invalidated = 0 + invalidation_failures = 0 + cache_roots: list[Path] = [] + for namespace in PROFILE_CACHE_NAMESPACES.values(): + try: + cache_roots.append(validated_cache_root(settings, namespace)) + except PreparedPlanError: + invalidation_failures += 1 + for cache_root in cache_roots: + if not cache_root.is_dir(): + continue + for cache_path in cache_root.rglob("*.npz"): + try: + with np.load(cache_path, allow_pickle=False) as payload: + _ = tuple(payload.files) + except (OSError, ValueError, EOFError): + try: + cache_path.unlink(missing_ok=True) + except OSError: + invalidation_failures += 1 + else: + invalidated += 1 + _record( + RepairRecord( + "invalidate_derived_cache", + ",".join(str(path) for path in cache_roots), + invalidation_failures == 0, + ( + f"Invalidated {invalidated} corrupt derived cache entrie(s); " + f"failures={invalidation_failures}." + ), + ) + ) + + from ser._internal.data.dataset_prepare import ( # noqa: TID251 + prepare_from_registry_entry, + ) + from ser._internal.data.dataset_registry import load_dataset_registry # noqa: TID251 + from ser._internal.data.label_ontology import resolve_label_ontology # noqa: TID251 + + registry = load_dataset_registry(settings=settings) + ontology = resolve_label_ontology(settings) + for entry in sorted(registry.values(), key=lambda item: item.dataset_id): + if entry.manifest_path.expanduser().is_file(): + continue + try: + built = prepare_from_registry_entry(settings=settings, entry=entry, ontology=ontology) + succeeded = bool(built) and all(path.is_file() for path in built) + detail = ( + f"Built {len(built)} manifest(s)." if succeeded else "No manifest was produced." + ) + except (OSError, RuntimeError, ValueError) as error: + succeeded = False + detail = str(error)[:2000] + _record(RepairRecord("rebuild_manifest", entry.dataset_id, succeeded, detail)) + + registry_snapshot = collect_dataset_registry_snapshot(settings=settings) + lfs_dataset_ids = { + issue.dataset_id + for issue in registry_snapshot.issues + if "lfs" in issue.code.lower() or "lfs" in issue.message.lower() + } + git_roots = { + root + for entry in registry.values() + if entry.dataset_id in lfs_dataset_ids + and (root := _git_root(entry.dataset_root)) is not None + } + for root in sorted(git_roots, key=str): + succeeded, detail = _run_repair_command(("git", "-C", str(root), "lfs", "checkout")) + if not succeeded and _network_repairs_allowed(): + pulled, pull_detail = _run_repair_command(("git", "-C", str(root), "lfs", "pull")) + if pulled: + succeeded, detail = _run_repair_command(("git", "-C", str(root), "lfs", "checkout")) + else: + detail = pull_detail + _record(RepairRecord("hydrate_git_lfs", str(root), succeeded, detail)) + logger.info( + "REPAIR_DONE actions=%d failures=%d elapsed=%.1fs", + len(records), + sum(not record.succeeded for record in records), + time.perf_counter() - started_at, + ) + return records + + +@contextmanager +def _network_repair_deadline(seconds: float) -> Iterator[None]: + """Hard-bounds a main-thread network repair or rejects unsupported execution.""" + if threading.current_thread() is not threading.main_thread(): + raise RuntimeError("Network repair requires main-thread hard-deadline support.") + if not hasattr(signal, "SIGALRM") or not hasattr(signal, "ITIMER_REAL"): + raise RuntimeError("Network repair hard-deadline support is unavailable.") + previous_handler = signal.getsignal(signal.SIGALRM) + + def _raise_timeout(_signum: int, _frame: object) -> None: + raise TimeoutError(f"Pinned model repair exceeded {seconds:.0f} seconds.") + + signal.signal(signal.SIGALRM, _raise_timeout) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0.0) + signal.signal(signal.SIGALRM, previous_handler) + + +def _repair_pinned_model(settings: AppConfig) -> RepairRecord: + """Redownloads only an explicitly revision-pinned Hugging Face model.""" + if not _network_repairs_allowed(): + return RepairRecord( + "redownload_pinned_model", + resolve_profile_name(settings), + False, + "Network policy denied repair; set SER_TRAINING_REPAIR_ALLOW_NETWORK=1 explicitly.", + ) + profile = resolve_profile_name(settings) + model_id = ( + settings.models.medium_model_id + if profile == "medium" + else settings.models.accurate_model_id if profile == "accurate" else "" + ) + if "@" not in model_id: + return RepairRecord( + "redownload_pinned_model", + model_id or profile, + False, + "Model repair requires an exact '@' pin.", + ) + repo_id, revision = model_id.rsplit("@", 1) + if not repo_id or not revision: + return RepairRecord("redownload_pinned_model", model_id, False, "Model pin is invalid.") + try: + from huggingface_hub import snapshot_download + from huggingface_hub.errors import HfHubHTTPError + except ImportError as error: + return RepairRecord("redownload_pinned_model", model_id, False, str(error)[:2000]) + try: + live_cache = settings.models.huggingface_cache_root.resolve(strict=False) + live_cache.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=".pinned-model-repair-", dir=live_cache.parent + ) as raw_staging: + staging_root = Path(raw_staging) + staging = staging_root / "cache" + if live_cache.is_dir(): + shutil.copytree(live_cache, staging) + else: + staging.mkdir() + with _network_repair_deadline(300.0): + resolved_snapshot = Path( + snapshot_download( + repo_id=repo_id, + revision=revision, + cache_dir=staging, + force_download=True, + etag_timeout=30, + max_workers=4, + ) + ) + if ( + not resolved_snapshot.resolve(strict=False).is_relative_to( + staging.resolve(strict=False) + ) + or not resolved_snapshot.is_dir() + ): + raise RuntimeError("Downloaded model snapshot did not validate inside staging.") + backup = Path( + tempfile.mkdtemp(prefix=f".{live_cache.name}.rollback-", dir=live_cache.parent) + ) + backup.rmdir() + moved_live = False + try: + if live_cache.exists(): + os.replace(live_cache, backup) + moved_live = True + os.replace(staging, live_cache) + except BaseException: + if moved_live and backup.exists() and not live_cache.exists(): + os.replace(backup, live_cache) + raise + else: + if backup.exists(): + shutil.rmtree(backup) + except (HfHubHTTPError, OSError, RuntimeError, TimeoutError, ValueError) as error: + return RepairRecord("redownload_pinned_model", model_id, False, str(error)[:2000]) + return RepairRecord( + "redownload_pinned_model", + model_id, + True, + "Pinned model snapshot redownloaded.", + ) + + +def run_training_readiness( + *, + settings: AppConfig, + load_utterances: Callable[[], list[Utterance] | None], + smoke_runner: SmokeRunner | None = None, + smoke_sample_cap: int = DEFAULT_SMOKE_SAMPLE_CAP, + repair: bool = False, + persist_quarantine_ledger: bool = True, + report_path: Path | None = None, + now: Callable[[], datetime] = lambda: datetime.now(UTC), +) -> tuple[ReadinessReport, tuple[Utterance, ...]]: + """Runs the shared deterministic readiness contract and atomically reports it.""" + started_at = time.perf_counter() + profile = resolve_profile_name(settings) + logger.info( + "READINESS_START profile=%s repair=%s smoke=%s report_path=%s", + profile, + repair, + smoke_runner is not None, + report_path or default_readiness_report_path(settings), + ) + created_at = now().astimezone(UTC).isoformat() + findings: list[ReadinessFinding] = [] + repairs: list[RepairRecord] = [] + resolved_utterances: tuple[Utterance, ...] = () + effective_utterances: tuple[Utterance, ...] = () + selected: tuple[Utterance, ...] = () + smoke: SmokeResult | None = None + quarantine_records: tuple[QuarantineRecord, ...] = () + destination = report_path or default_readiness_report_path(settings) + + try: + if repair: + repair_started_at = time.perf_counter() + repairs = _apply_pre_validation_repairs(settings) + repair_findings = [ + ReadinessFinding( + check="repair", + reason_code=FailureReasonCode.REPAIR_FAILED, + severity=FailureSeverity.ERROR, + message=f"Repair {repair_record.action!r} failed: {repair_record.message}", + blocking=True, + ) + for repair_record in repairs + if not repair_record.succeeded + ] + findings.extend(repair_findings) + repair_status = _readiness_status(repair_findings) + logger.log( + _readiness_log_level(repair_status), + "READINESS_PHASE_DONE phase=repair status=%s findings=%d elapsed=%.1fs", + repair_status, + len(repair_findings), + time.perf_counter() - repair_started_at, + ) + + phase_started_at = time.perf_counter() + logger.info("READINESS_PHASE_START phase=configuration") + configuration_findings = _configuration_findings(settings) + configuration_findings.extend(_restricted_backend_access_findings(settings)) + findings.extend(configuration_findings) + configuration_status = _readiness_status(configuration_findings) + logger.log( + _readiness_log_level(configuration_status), + "READINESS_PHASE_DONE phase=configuration status=%s findings=%d elapsed=%.1fs", + configuration_status, + len(configuration_findings), + time.perf_counter() - phase_started_at, + ) + + phase_started_at = time.perf_counter() + logger.info("READINESS_PHASE_START phase=dataset_registry") + registry_findings = _registry_findings(settings) + findings.extend(registry_findings) + registry_status = _readiness_status(registry_findings) + logger.log( + _readiness_log_level(registry_status), + "READINESS_PHASE_DONE phase=dataset_registry status=%s findings=%d elapsed=%.1fs", + registry_status, + len(registry_findings), + time.perf_counter() - phase_started_at, + ) + + phase_started_at = time.perf_counter() + logger.info("READINESS_PHASE_START phase=manifest_load") + try: + utterances = load_utterances() + except (OSError, RuntimeError, ValueError) as error: + utterances = None + findings.append( + ReadinessFinding( + check="manifest_integrity", + reason_code=( + FailureReasonCode.GIT_LFS_POINTER + if "git lfs" in str(error).lower() + else FailureReasonCode.MANIFEST_INVALID + ), + severity=FailureSeverity.ERROR, + message=str(error), + blocking=True, + scope=( + FailureScope.CORPUS if "git lfs" in str(error).lower() else FailureScope.RUN + ), + ) + ) + resolved_utterances = tuple(utterances or ()) + manifest_findings = [ + finding for finding in findings if finding.check == "manifest_integrity" + ] + if not resolved_utterances: + empty_dataset_finding = ReadinessFinding( + check="dataset_registry", + reason_code=FailureReasonCode.DATASET_NOT_FOUND, + severity=FailureSeverity.ERROR, + message="No effective training utterances were resolved.", + blocking=True, + ) + findings.append(empty_dataset_finding) + manifest_findings.append(empty_dataset_finding) + manifest_status = _readiness_status(manifest_findings) + logger.log( + _readiness_log_level(manifest_status), + "READINESS_PHASE_DONE phase=manifest_load status=%s utterances=%d findings=%d elapsed=%.1fs", + manifest_status, + len(resolved_utterances), + len(manifest_findings), + time.perf_counter() - phase_started_at, + ) + + effective_utterances = resolved_utterances + if resolved_utterances: + phase_started_at = time.perf_counter() + logger.info( + "READINESS_PHASE_START phase=dataset_media samples=%d", + len(resolved_utterances), + ) + roots_started_at = time.perf_counter() + logger.info( + "DATASET_MEDIA_ROOTS_START samples=%d", + len(resolved_utterances), + ) + allowed_roots = _allowed_media_roots(settings) + logger.info( + "DATASET_MEDIA_ROOTS_DONE roots=%d elapsed=%.1fs", + len(allowed_roots), + time.perf_counter() - roots_started_at, + ) + media_findings = _media_findings(resolved_utterances, allowed_roots=allowed_roots) + contained, quarantine_records, effective_utterances = ( + _contain_readiness_sample_failures( + settings=settings, + utterances=resolved_utterances, + findings=media_findings, + occurred_at=created_at, + allowed_roots=allowed_roots, + ) + ) + findings.extend(contained) + if quarantine_records and persist_quarantine_ledger: + write_quarantine_ledger( + settings.tmp_folder / f"quarantine-{profile}.jsonl", + quarantine_records, + ) + media_status = _readiness_status(contained) + logger.log( + _readiness_log_level(media_status), + "READINESS_PHASE_DONE phase=dataset_media status=%s effective=%d quarantined=%d findings=%d elapsed=%.1fs", + media_status, + len(effective_utterances), + len(quarantine_records), + len(contained), + time.perf_counter() - phase_started_at, + ) + + phase_started_at = time.perf_counter() + logger.info( + "READINESS_PHASE_START phase=split_feasibility samples=%d", + len(effective_utterances), + ) + split_findings = _split_findings(effective_utterances, settings=settings) + findings.extend(split_findings) + split_status = _readiness_status(split_findings) + logger.log( + _readiness_log_level(split_status), + "READINESS_PHASE_DONE phase=split_feasibility status=%s findings=%d elapsed=%.1fs", + split_status, + len(split_findings), + time.perf_counter() - phase_started_at, + ) + + phase_started_at = time.perf_counter() + logger.info( + "READINESS_PHASE_START phase=filesystem_resources effective=%d", + len(effective_utterances), + ) + resource_findings = _resource_findings(settings, len(effective_utterances)) + findings.extend(resource_findings) + resource_status = _readiness_status(resource_findings) + logger.log( + _readiness_log_level(resource_status), + "READINESS_PHASE_DONE phase=filesystem_resources status=%s findings=%d elapsed=%.1fs", + resource_status, + len(resource_findings), + time.perf_counter() - phase_started_at, + ) + + phase_started_at = time.perf_counter() + logger.info("READINESS_PHASE_START phase=backend_smoke") + try: + selected = ( + select_smoke_samples(effective_utterances, cap=smoke_sample_cap) + if effective_utterances + else () + ) + except ValueError as error: + findings.append( + ReadinessFinding( + check="backend_smoke_selection", + reason_code=FailureReasonCode.INVALID_CONFIGURATION, + severity=FailureSeverity.ERROR, + message=str(error), + blocking=True, + ) + ) + smoke_findings_start = len(findings) + if smoke_runner is not None and not any(finding.blocking for finding in findings): + probe_parent = settings.tmp_folder + probe_parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="ser-smoke-", dir=probe_parent) as probe_dir: + try: + smoke = smoke_runner( + settings=settings, + samples=selected, + probe_cache_dir=Path(probe_dir), + ) + if ( + smoke.attempted != len(selected) + or smoke.succeeded != smoke.attempted + or smoke.feature_dim <= 0 + or not smoke.cache_round_trip + ): + raise ValueError( + "Backend smoke result violates the feature/cache contract." + ) + except (ImportError, OSError, RuntimeError, ValueError) as error: + findings.append( + ReadinessFinding( + check="backend_smoke", + reason_code=_backend_failure_reason(error), + severity=FailureSeverity.ERROR, + message=str(error), + blocking=True, + ) + ) + smoke_findings = findings[smoke_findings_start:] + smoke_status = _readiness_status(smoke_findings) + logger.log( + _readiness_log_level(smoke_status), + "READINESS_PHASE_DONE phase=backend_smoke status=%s selected=%d findings=%d elapsed=%.1fs", + smoke_status, + len(selected), + len(smoke_findings), + time.perf_counter() - phase_started_at, + ) + + phase_started_at = time.perf_counter() + logger.info("READINESS_PHASE_START phase=inventory_digest") + if effective_utterances and not any( + finding.reason_code + in { + FailureReasonCode.MEDIA_NOT_REGULAR, + FailureReasonCode.MANIFEST_INVALID, + FailureReasonCode.GIT_LFS_POINTER, + } + and finding.blocking + for finding in findings + ): + registry_digest, manifest_digest, media_digest = _inventory_digests( + settings, + resolved_utterances, + media_utterances=resolved_utterances, + ) + else: + snapshot = collect_dataset_registry_snapshot(settings=settings) + registry_digest = digest_payload( + { + "entries": [asdict(item) for item in snapshot.entries], + "issues": [asdict(item) for item in snapshot.issues], + } + ) + manifest_digest = digest_payload([]) + media_digest = digest_payload([]) + logger.info( + "READINESS_PHASE_DONE phase=inventory_digest status=PASS elapsed=%.1fs", + time.perf_counter() - phase_started_at, + ) + + report = ReadinessReport( + schema_version=READINESS_SCHEMA_VERSION, + created_at=created_at, + profile=profile, + settings_digest=settings_digest(settings), + registry_digest=registry_digest, + manifest_digest=manifest_digest, + media_digest=media_digest, + selected_sample_ids=tuple(item.sample_id for item in selected), + findings=tuple(findings), + repairs=tuple(repairs), + smoke=smoke, + quarantined_samples=len(quarantine_records), + quarantine=quarantine_records, + effective_sample_ids=tuple(item.sample_id for item in effective_utterances), + ) + if repair and any( + finding.reason_code is FailureReasonCode.BACKEND_UNAVAILABLE and finding.blocking + for finding in findings + ): + logger.info("REPAIR_START action=redownload_pinned_model") + model_repair = _repair_pinned_model(settings) + _log_repair_record(model_repair) + revalidated, revalidated_utterances = run_training_readiness( + settings=settings, + load_utterances=load_utterances, + smoke_runner=smoke_runner, + smoke_sample_cap=smoke_sample_cap, + repair=False, + report_path=destination, + now=now, + ) + report = replace( + revalidated, + repairs=tuple([*repairs, model_repair]), + ) + atomic_write_json(destination, report.to_dict()) + logger.info( + "READINESS_DONE profile=%s ready=%s findings=%d repairs=%d report_path=%s elapsed=%.1fs", + report.profile, + report.ready, + len(report.findings), + len(report.repairs), + destination, + time.perf_counter() - started_at, + ) + return report, revalidated_utterances + atomic_write_json(destination, report.to_dict()) + logger.info( + "READINESS_DONE profile=%s ready=%s findings=%d repairs=%d report_path=%s elapsed=%.1fs", + report.profile, + report.ready, + len(report.findings), + len(report.repairs), + destination, + time.perf_counter() - started_at, + ) + return report, effective_utterances + except Exception: + logger.exception( + "READINESS_DONE profile=%s ready=false status=ERROR elapsed=%.1fs", + profile, + time.perf_counter() - started_at, + ) + raise + + +def ensure_training_ready( + *, + settings: AppConfig, + load_utterances: Callable[[], list[Utterance] | None], + smoke_runner: SmokeRunner | None = None, + repair: bool = False, + report_path: Path | None = None, +) -> tuple[ReadinessReport, tuple[Utterance, ...]]: + """Runs readiness and raises a typed validation error on any blocking finding.""" + report, utterances = run_training_readiness( + settings=settings, + load_utterances=load_utterances, + smoke_runner=smoke_runner, + repair=repair, + report_path=report_path, + ) + if not report.ready: + raise TrainingReadinessError(report) + return report, utterances + + +def build_prepared_plan( + *, + settings: AppConfig, + readiness: ReadinessReport, + backend_id: str, + model_id: str, + model_revision: str, + device: str, + dtype: str, + recipe_digest: str, + split_ledger_digest: str, + quarantine_ledger_digest: str, + cache_namespace: str, + cache_version: str, + cache_keys: Sequence[str], + effective_counts: Mapping[str, Mapping[str, int]], + feature_shape: tuple[int, int], + feature_dtype: str, + windowing_policy: Mapping[str, object], + noise_statistics: Mapping[str, object], + payload_path: Path, + package_version: str, + sample_ledger: Sequence[Mapping[str, object]] = (), + window_ledger: Sequence[Mapping[str, object]] = (), + disposition_counts: Mapping[str, int] | None = None, + created_at: str | None = None, +) -> PreparedPlan: + """Builds and signs a complete prepared-plan document.""" + if not readiness.ready: + raise PreparedPlanError("Cannot build a prepared plan from failed readiness validation.") + if not payload_path.is_file(): + raise PreparedPlanError(f"Prepared feature payload does not exist: {payload_path}") + plan = PreparedPlan( + schema_version=PREPARED_PLAN_SCHEMA_VERSION, + created_at=created_at or datetime.now(UTC).isoformat(), + code_version=PREPARATION_CODE_VERSION, + package_version=package_version, + profile=resolve_profile_name(settings), + backend_id=backend_id, + model_id=model_id, + model_revision=model_revision, + device=device, + dtype=dtype, + settings_digest=readiness.settings_digest, + registry_digest=readiness.registry_digest, + manifest_digest=readiness.manifest_digest, + media_digest=readiness.media_digest, + recipe_digest=recipe_digest, + split_ledger_digest=split_ledger_digest, + quarantine_ledger_digest=quarantine_ledger_digest, + cache_namespace=cache_namespace, + cache_version=cache_version, + cache_keys=tuple(sorted(cache_keys)), + effective_counts=effective_counts, + sample_ledger=tuple(sample_ledger), + window_ledger=tuple(window_ledger), + disposition_counts=dict(disposition_counts or {}), + feature_shape=feature_shape, + feature_dtype=feature_dtype, + windowing_policy=windowing_policy, + noise_statistics=noise_statistics, + validation_findings=tuple(finding.to_dict() for finding in readiness.findings), + repairs=tuple(repair.to_dict() for repair in readiness.repairs), + payload_path=str(payload_path.resolve()), + payload_digest=hash_file(payload_path), + ) + return replace(plan, overall_digest=digest_payload(plan.unsigned_dict())) + + +def write_prepared_plan(path: Path, plan: PreparedPlan) -> None: + """Validates and atomically publishes a complete prepared plan.""" + if plan.overall_digest != digest_payload(plan.unsigned_dict()): + raise PreparedPlanError("Prepared plan overall digest is invalid.") + atomic_write_json(path, plan.to_dict()) + + +def _validate_json_value(value: object, *, location: str) -> None: + """Exhaustively validates one untrusted JSON subtree.""" + if value is None or isinstance(value, str | bool): + return + if isinstance(value, int) and not isinstance(value, bool): + return + if isinstance(value, float): + if math.isfinite(value): + return + raise PreparedPlanError(f"Prepared plan {location} contains a non-finite number.") + if isinstance(value, list): + for index, item in enumerate(value): + _validate_json_value(item, location=f"{location}[{index}]") + return + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str) or not key: + raise PreparedPlanError(f"Prepared plan {location} has an invalid key.") + _validate_json_value(item, location=f"{location}.{key}") + return + raise PreparedPlanError( + f"Prepared plan {location} has unsupported type {type(value).__name__}." + ) + + +def _validate_prepared_plan_mapping(raw: dict[object, object]) -> dict[str, object]: + """Validates every prepared-plan field before typed construction.""" + expected = { + "schema_version", + "created_at", + "code_version", + "package_version", + "profile", + "backend_id", + "model_id", + "model_revision", + "device", + "dtype", + "settings_digest", + "registry_digest", + "manifest_digest", + "media_digest", + "recipe_digest", + "split_ledger_digest", + "quarantine_ledger_digest", + "cache_namespace", + "cache_version", + "cache_keys", + "effective_counts", + "sample_ledger", + "window_ledger", + "disposition_counts", + "feature_shape", + "feature_dtype", + "windowing_policy", + "noise_statistics", + "validation_findings", + "repairs", + "payload_path", + "payload_digest", + "overall_digest", + } + if any(not isinstance(key, str) for key in raw): + raise PreparedPlanError("Prepared plan keys must all be strings.") + normalized = cast(dict[str, object], raw) + missing = expected - normalized.keys() + unknown = normalized.keys() - expected + if missing or unknown: + raise PreparedPlanError( + f"Prepared plan fields differ from schema: missing={sorted(missing)} unknown={sorted(unknown)}." + ) + _validate_json_value(normalized, location="root") + if not isinstance(normalized["schema_version"], int) or isinstance( + normalized["schema_version"], bool + ): + raise PreparedPlanError("Prepared plan schema_version must be an integer.") + string_fields = expected - { + "schema_version", + "cache_keys", + "effective_counts", + "sample_ledger", + "window_ledger", + "disposition_counts", + "feature_shape", + "windowing_policy", + "noise_statistics", + "validation_findings", + "repairs", + } + for field_name in string_fields: + value = normalized[field_name] + if not isinstance(value, str) or not value.strip(): + raise PreparedPlanError(f"Prepared plan {field_name} must be a non-empty string.") + if normalized["profile"] not in PROFILE_CACHE_NAMESPACES: + raise PreparedPlanError("Prepared plan profile is unsupported.") + for field_name in ("cache_keys",): + value = normalized[field_name] + if not isinstance(value, list) or not all( + isinstance(item, str) and bool(item) for item in value + ): + raise PreparedPlanError(f"Prepared plan {field_name} must be a string list.") + for field_name in ("sample_ledger", "window_ledger", "validation_findings", "repairs"): + value = normalized[field_name] + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise PreparedPlanError(f"Prepared plan {field_name} must be an object list.") + for field_name in ( + "effective_counts", + "disposition_counts", + "windowing_policy", + "noise_statistics", + ): + if not isinstance(normalized[field_name], dict): + raise PreparedPlanError(f"Prepared plan {field_name} must be an object.") + counts = cast(dict[str, object], normalized["effective_counts"]) + if not all( + isinstance(group, dict) + and all( + isinstance(key, str) + and isinstance(count, int) + and not isinstance(count, bool) + and count >= 0 + for key, count in group.items() + ) + for group in counts.values() + ): + raise PreparedPlanError( + "Prepared plan effective_counts must contain non-negative integers." + ) + dispositions = cast(dict[str, object], normalized["disposition_counts"]) + if not all( + isinstance(key, str) + and isinstance(count, int) + and not isinstance(count, bool) + and count >= 0 + for key, count in dispositions.items() + ): + raise PreparedPlanError("Prepared plan disposition_counts is invalid.") + shape = normalized["feature_shape"] + if ( + not isinstance(shape, list) + or len(shape) != 2 + or not all( + isinstance(item, int) and not isinstance(item, bool) and item > 0 for item in shape + ) + ): + raise PreparedPlanError("Prepared plan feature_shape must contain two positive integers.") + if not Path(cast(str, normalized["payload_path"])).is_absolute(): + raise PreparedPlanError("Prepared plan payload_path must be absolute.") + return normalized + + +def load_prepared_plan(path: Path) -> PreparedPlan: + """Loads one prepared plan using explicit boundary validation.""" + try: + raw = json.loads( + path.read_text(encoding="utf-8"), + parse_constant=lambda value: (_ for _ in ()).throw( + ValueError(f"non-finite JSON constant {value}") + ), + ) + except (OSError, json.JSONDecodeError) as error: + raise PreparedPlanError(f"Unable to read prepared plan {path}: {error}") from error + if not isinstance(raw, dict): + raise PreparedPlanError("Prepared plan root must be a JSON object.") + validated_raw = _validate_prepared_plan_mapping(raw) + try: + raw_feature_shape = cast(list[object], validated_raw["feature_shape"]) + if not isinstance(raw_feature_shape, list) or len(raw_feature_shape) != 2: + raise ValueError("feature_shape must contain exactly two integers") + plan = PreparedPlan( + schema_version=cast(int, validated_raw["schema_version"]), + created_at=cast(str, validated_raw["created_at"]), + code_version=str(raw["code_version"]), + package_version=str(raw["package_version"]), + profile=cast(ProfileName, raw["profile"]), + backend_id=str(raw["backend_id"]), + model_id=str(raw["model_id"]), + model_revision=str(raw["model_revision"]), + device=str(raw["device"]), + dtype=str(raw["dtype"]), + settings_digest=str(raw["settings_digest"]), + registry_digest=str(raw["registry_digest"]), + manifest_digest=str(raw["manifest_digest"]), + media_digest=str(raw["media_digest"]), + recipe_digest=str(raw["recipe_digest"]), + split_ledger_digest=str(raw["split_ledger_digest"]), + quarantine_ledger_digest=str(raw["quarantine_ledger_digest"]), + cache_namespace=str(raw["cache_namespace"]), + cache_version=str(raw["cache_version"]), + cache_keys=tuple(str(item) for item in cast(list[object], raw["cache_keys"])), + effective_counts=cast(Mapping[str, Mapping[str, int]], raw["effective_counts"]), + sample_ledger=tuple(cast(list[Mapping[str, object]], raw["sample_ledger"])), + window_ledger=tuple(cast(list[Mapping[str, object]], raw["window_ledger"])), + disposition_counts=cast(Mapping[str, int], raw["disposition_counts"]), + feature_shape=(cast(int, raw_feature_shape[0]), cast(int, raw_feature_shape[1])), + feature_dtype=str(raw["feature_dtype"]), + windowing_policy=cast(Mapping[str, object], raw["windowing_policy"]), + noise_statistics=cast(Mapping[str, object], raw["noise_statistics"]), + validation_findings=tuple(cast(list[Mapping[str, object]], raw["validation_findings"])), + repairs=tuple(cast(list[Mapping[str, object]], raw["repairs"])), + payload_path=str(raw["payload_path"]), + payload_digest=str(raw["payload_digest"]), + overall_digest=str(raw["overall_digest"]), + ) + except (KeyError, TypeError, ValueError) as error: + raise PreparedPlanError(f"Prepared plan schema is invalid: {error}") from error + if plan.schema_version != PREPARED_PLAN_SCHEMA_VERSION: + raise PreparedPlanError(f"Unsupported prepared plan schema {plan.schema_version}.") + if plan.code_version != PREPARATION_CODE_VERSION: + raise PreparedPlanError("Prepared plan code/schema version is stale.") + if plan.overall_digest != digest_payload(plan.unsigned_dict()): + raise PreparedPlanError("Prepared plan overall digest mismatch.") + payload_path = Path(plan.payload_path) + if not payload_path.is_file() or hash_file(payload_path) != plan.payload_digest: + raise PreparedPlanError("Prepared feature payload is missing or digest-invalid.") + return plan + + +def validate_prepared_plan( + plan: PreparedPlan, + *, + settings: AppConfig, + readiness: ReadinessReport, + backend_id: str, + model_id: str, + model_revision: str, + device: str, + dtype: str, + recipe_digest: str, + split_ledger_digest: str, + quarantine_ledger_digest: str, + cache_namespace: str, + cache_version: str, + cache_keys: Sequence[str], + package_version: str, +) -> None: + """Rejects any prepared plan whose checked training inputs have changed.""" + expected: dict[str, object] = { + "profile": resolve_profile_name(settings), + "backend_id": backend_id, + "model_id": model_id, + "model_revision": model_revision, + "device": device, + "dtype": dtype, + "settings_digest": readiness.settings_digest, + "registry_digest": readiness.registry_digest, + "manifest_digest": readiness.manifest_digest, + "media_digest": readiness.media_digest, + "recipe_digest": recipe_digest, + "split_ledger_digest": split_ledger_digest, + "quarantine_ledger_digest": quarantine_ledger_digest, + "cache_namespace": cache_namespace, + "cache_version": cache_version, + "cache_keys": tuple(sorted(cache_keys)), + "package_version": package_version, + } + mismatches = [name for name, value in expected.items() if getattr(plan, name) != value] + if mismatches: + raise PreparedPlanError( + "Prepared plan is stale for: " + ", ".join(sorted(mismatches)) + "." + ) + + +def effective_counts(utterances: Sequence[Utterance]) -> dict[str, dict[str, int]]: + """Returns deterministic effective counts by required reporting dimensions.""" + dimensions: dict[str, Counter[str]] = { + "corpus": Counter(item.corpus for item in utterances), + "class": Counter(item.label or "unknown" for item in utterances), + "language": Counter(item.language or "unknown" for item in utterances), + "split": Counter(str(item.split or "unspecified") for item in utterances), + "disposition": Counter({"included": len(utterances)}), + } + return {name: dict(sorted(counter.items())) for name, counter in dimensions.items()} + + +__all__ = [ + "DEFAULT_SMOKE_SAMPLE_CAP", + "FailureClassification", + "FailureDisposition", + "FailureReasonCode", + "FailureScope", + "FailureSeverity", + "PreparedPlan", + "PreparedPlanError", + "QuarantineBudgetExceeded", + "QuarantinePolicy", + "QuarantineRecord", + "ReadinessFinding", + "ReadinessReport", + "SmokeResult", + "TrainingMode", + "TrainingOperation", + "TrainingReadinessError", + "atomic_write_json", + "build_prepared_plan", + "build_quarantine_record", + "canonical_json_bytes", + "classify_failure", + "default_prepared_payload_path", + "default_prepared_plan_path", + "default_readiness_report_path", + "digest_payload", + "effective_counts", + "enforce_quarantine_budget", + "ensure_training_ready", + "hash_file", + "load_prepared_plan", + "run_training_readiness", + "select_smoke_samples", + "settings_digest", + "validate_prepared_plan", + "write_prepared_plan", + "write_quarantine_ledger", +] diff --git a/tests/suites/unit/models/test_training_readiness.py b/tests/suites/unit/models/test_training_readiness.py new file mode 100644 index 0000000..c8349b5 --- /dev/null +++ b/tests/suites/unit/models/test_training_readiness.py @@ -0,0 +1,908 @@ +"""Regression contracts for robust training readiness and prepared plans.""" + +from __future__ import annotations + +import logging +import sys +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path +from types import MappingProxyType, ModuleType +from typing import cast + +import numpy as np +import pytest +import soundfile as sf +from hypothesis import given +from hypothesis import strategies as st + +from ser._internal.config.schema import ( # noqa: TID251 + DataLoaderConfig, + DatasetConfig, + ModelsConfig, + RuntimeFlags, +) +from ser._internal.data.manifest import SplitName, Utterance # noqa: TID251 +from ser._internal.models.training_readiness import ( # noqa: TID251 + CacheEntryCorruptError, + FailureDisposition, + FailureReasonCode, + FailureScope, + OptionalArtifactError, + PreparedPlan, + PreparedPlanError, + QuarantineBudgetExceeded, + QuarantinePolicy, + ReadinessReport, + RepairRecord, + SmokeResult, + TrainingMode, + TrainingOperation, + WindowContainmentError, + atomic_write_json, + build_prepared_plan, + build_quarantine_record, + canonical_json_bytes, + classify_failure, + digest_payload, + enforce_quarantine_budget, + load_prepared_plan, + quarantine_ledger_digest, + run_training_readiness, + select_smoke_samples, + settings_digest, + validate_prepared_plan, + write_prepared_plan, + write_quarantine_ledger, +) +from ser._internal.utils.audio_utils import ( # noqa: TID251 + AudioDecodeError, + AudioIntegrityError, +) +from ser.config import AppConfig + + +def _settings(tmp_path: Path, *, ratio: float = 0.5) -> AppConfig: + return AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + data_loader=DataLoaderConfig( + max_workers=1, + max_failed_file_ratio=ratio, + max_failed_files=4, + max_failed_file_ratio_per_corpus=ratio, + max_failed_file_ratio_per_class=ratio, + max_failures_per_reason=2, + min_remaining_per_class_split=1, + ), + models=ModelsConfig( + folder=tmp_path / "models", + model_cache_dir=tmp_path / "cache", + ), + ) + + +def _utterance( + tmp_path: Path, + sample_id: str, + label: str, + *, + corpus: str = "fixture", + split: str | None = None, + suffix: str = ".wav", + value: float = 0.1, +) -> Utterance: + path = tmp_path / f"{sample_id}{suffix}" + sf.write(path, np.full(800, value, dtype=np.float32), 8000) + return Utterance( + schema_version=1, + sample_id=sample_id, + corpus=corpus, + audio_path=path, + label=label, + language="en", + split=cast(SplitName | None, split), + ) + + +def _inventory(tmp_path: Path) -> list[Utterance]: + return [ + _utterance(tmp_path, "calm-1", "calm", value=0.10), + _utterance(tmp_path, "calm-2", "calm", value=0.11), + _utterance(tmp_path, "calm-3", "calm", value=0.12), + _utterance(tmp_path, "calm-4", "calm", value=0.13), + _utterance(tmp_path, "happy-1", "happy", value=0.20), + _utterance(tmp_path, "happy-2", "happy", value=0.21), + _utterance(tmp_path, "happy-3", "happy", value=0.22), + _utterance(tmp_path, "happy-4", "happy", value=0.23), + ] + + +def test_training_operation_rejects_unsafe_flag_combinations(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="repair"): + TrainingOperation(mode=TrainingMode.TRAIN, repair=True).validate() + with pytest.raises(ValueError, match="prepared-plan"): + TrainingOperation( + mode=TrainingMode.PREPARE_ONLY, + prepared_plan=tmp_path / "plan.json", + ).validate() + + +def test_smoke_selection_is_bounded_deterministic_and_stratified(tmp_path: Path) -> None: + samples = _inventory(tmp_path) + segmented = replace( + samples[0], + sample_id="calm-segment", + corpus="other", + start_seconds=0.0, + duration_seconds=0.05, + ) + first = select_smoke_samples([*samples, segmented], cap=3) + second = select_smoke_samples([segmented, *reversed(samples)], cap=3) + assert [item.sample_id for item in first] == [item.sample_id for item in second] + assert len(first) == 3 + assert {item.corpus for item in first} == {"fixture", "other"} + + +def test_valid_tiny_audio_inventory_passes_and_writes_atomic_report(tmp_path: Path) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + report_path = tmp_path / "reports" / "readiness.json" + + report, resolved = run_training_readiness( + settings=settings, + load_utterances=lambda: samples, + report_path=report_path, + now=lambda: datetime(2026, 1, 1, tzinfo=UTC), + ) + + assert report.ready is True + assert resolved == tuple(samples) + assert report_path.is_file() + assert not list(report_path.parent.glob(f".{report_path.name}.*")) + + +def test_readiness_logs_start_phase_and_done_events( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Dry-run readiness should emit visible progress before long work returns.""" + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + caplog.set_level(logging.INFO, logger="ser._internal.models.training_readiness") + + report, _ = run_training_readiness(settings=settings, load_utterances=lambda: samples) + + messages = [record.getMessage() for record in caplog.records] + assert report.ready is True + assert any(message.startswith("READINESS_START") for message in messages) + assert any("READINESS_PHASE_START phase=manifest_load" in message for message in messages) + assert any(message.startswith("DATASET_MEDIA_ROOTS_START") for message in messages) + assert any(message.startswith("DATASET_MEDIA_ROOTS_DONE") for message in messages) + assert any(message.startswith("DATASET_MEDIA_CHECK_START") for message in messages) + assert any(message.startswith("DATASET_MEDIA_PROGRESS") for message in messages) + assert any(message.startswith("DATASET_MEDIA_CHECK_DONE") for message in messages) + assert any(message.startswith("DATASET_MEDIA_CONTAINMENT_START") for message in messages) + assert any(message.startswith("DATASET_MEDIA_CONTAINMENT_DONE") for message in messages) + assert any("READINESS_PHASE_DONE phase=dataset_media" in message for message in messages) + assert any(message.startswith("INVENTORY_DIGEST_START") for message in messages) + assert any(message.startswith("INVENTORY_DIGEST_PROGRESS") for message in messages) + assert any(message.startswith("INVENTORY_DIGEST_DONE") for message in messages) + assert any(message.startswith("READINESS_DONE") for message in messages) + + +def test_missing_media_is_contained_before_backend_smoke(tmp_path: Path) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + missing = replace(samples[0], audio_path=tmp_path / "missing.wav") + smoke_called = False + + def _smoke(**_kwargs: object) -> SmokeResult: + nonlocal smoke_called + smoke_called = True + sample_count = len(cast(tuple[Utterance, ...], _kwargs["samples"])) + return SmokeResult( + sample_count, + sample_count, + 3, + True, + "handcrafted", + "builtin", + "cpu", + "float64", + ) + + report, _ = run_training_readiness( + settings=settings, + load_utterances=lambda: [missing, *samples[1:]], + smoke_runner=_smoke, + ) + + assert report.ready is True + assert smoke_called is True + assert FailureReasonCode.MEDIA_MISSING in {item.reason_code for item in report.findings} + + +def test_one_isolated_corrupt_sample_is_quarantined_and_revalidated(tmp_path: Path) -> None: + settings = _settings(tmp_path, ratio=0.5) + samples = [ + *_inventory(tmp_path), + _utterance(tmp_path, "calm-5", "calm", value=0.14), + _utterance(tmp_path, "happy-5", "happy", value=0.24), + ] + samples[0].audio_path.write_bytes(b"not an audio container") + + report, effective = run_training_readiness( + settings=settings, + load_utterances=lambda: samples, + ) + + assert report.ready is True + assert [record.sample_id for record in report.quarantine] == [samples[0].sample_id] + assert samples[0].sample_id not in {item.sample_id for item in effective} + assert any( + finding.reason_code is FailureReasonCode.MEDIA_DECODE_FAILED and not finding.blocking + for finding in report.findings + ) + + +def test_dry_run_readiness_keeps_quarantine_only_in_atomic_report(tmp_path: Path) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + samples[0].audio_path.write_bytes(b"not an audio container") + report_path = tmp_path / "dry-run-readiness.json" + report, _ = run_training_readiness( + settings=settings, + load_utterances=lambda: samples, + persist_quarantine_ledger=False, + report_path=report_path, + ) + assert report.quarantine + assert report_path.is_file() + assert not (settings.tmp_folder / "quarantine-fast.jsonl").exists() + + +def test_permission_and_resource_os_errors_never_quarantine() -> None: + for error in (PermissionError("denied"), OSError(28, "disk full")): + classification = classify_failure(error, scope=FailureScope.SAMPLE) + assert classification.disposition is FailureDisposition.ABORT + + +def test_duplicate_pcm_is_detected_across_audio_containers(tmp_path: Path) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + pcm = np.arange(-400, 400, dtype=np.int16) + sf.write(samples[0].audio_path, pcm, 8000, subtype="PCM_16") + flac_path = tmp_path / "duplicate.flac" + sf.write(flac_path, pcm, 8000, format="FLAC", subtype="PCM_16") + samples[1] = replace(samples[1], audio_path=flac_path) + + report, _ = run_training_readiness(settings=settings, load_utterances=lambda: samples) + + duplicates = [ + finding + for finding in report.findings + if finding.reason_code is FailureReasonCode.DUPLICATE_CONTENT + ] + assert len(duplicates) == 1 + assert duplicates[0].blocking is True + + +def test_media_path_escape_blocks_before_metadata_or_hash_access(tmp_path: Path) -> None: + settings = _settings(tmp_path) + outside_path = tmp_path.parent / f"{tmp_path.name}-outside.wav" + sf.write(outside_path, np.ones(80, dtype=np.float32), 8000) + outside = replace(_utterance(tmp_path, "outside", "calm", value=0.4), audio_path=outside_path) + samples = [outside, *_inventory(tmp_path)] + report, _ = run_training_readiness(settings=settings, load_utterances=lambda: samples) + assert any( + finding.reason_code is FailureReasonCode.MANIFEST_INVALID and finding.sample_id == "outside" + for finding in report.findings + ) + + +def test_smoke_selection_rejects_an_unsatisfiable_strata_cap(tmp_path: Path) -> None: + samples = [ + _utterance(tmp_path, "a", "calm", corpus="one", value=0.1), + _utterance(tmp_path, "b", "happy", corpus="two", value=0.2), + ] + with pytest.raises(ValueError, match="cannot cover"): + select_smoke_samples(samples, cap=1) + + +def test_repair_cleans_owned_staging_and_corrupt_cache_then_revalidates( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + settings.tmp_folder.mkdir(parents=True) + staging = settings.tmp_folder / ".ser-write-probe-abandoned" + staging.write_text("probe", encoding="utf-8") + corrupt_cache = settings.tmp_folder / "medium_embeddings" / "broken.npz" + corrupt_cache.parent.mkdir(parents=True) + corrupt_cache.write_bytes(b"broken") + user_npz = settings.tmp_folder / "user-data" / "important.npz" + user_npz.parent.mkdir(parents=True) + user_npz.write_bytes(b"not an application cache") + caplog.set_level(logging.INFO, logger="ser._internal.models.training_readiness") + + report, _ = run_training_readiness( + settings=settings, + load_utterances=lambda: samples, + repair=True, + ) + + assert report.ready is True + assert not staging.exists() + assert not corrupt_cache.exists() + assert user_npz.read_bytes() == b"not an application cache" + actions = {record.action for record in report.repairs} + assert { + "create_application_directory", + "clean_application_staging", + "invalidate_derived_cache", + } <= actions + messages = [record.getMessage() for record in caplog.records] + assert any(message.startswith("REPAIR_START") for message in messages) + assert any("REPAIR_ACTION action=clean_application_staging" in message for message in messages) + assert any(message.startswith("REPAIR_DONE") for message in messages) + + +def test_backend_repair_is_followed_by_complete_revalidation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + attempts = 0 + + def _smoke(**_kwargs: object) -> SmokeResult: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ImportError("backend dependency unavailable") + sample_count = len(cast(tuple[Utterance, ...], _kwargs["samples"])) + return SmokeResult( + sample_count, + sample_count, + 3, + True, + "handcrafted", + "builtin", + "cpu", + "float64", + ) + + monkeypatch.setattr( + "ser._internal.models.training_readiness._repair_pinned_model", + lambda _settings: RepairRecord("redownload_pinned_model", "fixture", True, "repaired"), + ) + report, _ = run_training_readiness( + settings=settings, + load_utterances=lambda: samples, + smoke_runner=_smoke, + repair=True, + ) + assert attempts == 2 + assert report.ready is True + assert any(record.action == "redownload_pinned_model" for record in report.repairs) + + +def test_pinned_model_repair_rolls_back_atomic_cache_publication( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import ser._internal.models.training_readiness as readiness_module # noqa: TID251 + + base = _settings(tmp_path) + settings = replace( + base, + models=replace(base.models, medium_model_id="owner/model@deadbeef"), + runtime_flags=RuntimeFlags(profile_pipeline=True, medium_profile=True), + ) + live_cache = settings.models.huggingface_cache_root + live_cache.mkdir(parents=True) + sentinel = live_cache / "valid.bin" + sentinel.write_bytes(b"preserve") + hub = ModuleType("huggingface_hub") + errors = ModuleType("huggingface_hub.errors") + + class _HubError(Exception): + pass + + def _snapshot_download(**kwargs: object) -> str: + cache_dir = Path(cast(Path, kwargs["cache_dir"])) + snapshot = cache_dir / "snapshots" / "deadbeef" + snapshot.mkdir(parents=True) + (snapshot / "weights.bin").write_bytes(b"new") + return str(snapshot) + + hub.__dict__["snapshot_download"] = _snapshot_download + errors.__dict__["HfHubHTTPError"] = _HubError + monkeypatch.setitem(sys.modules, "huggingface_hub", hub) + monkeypatch.setitem(sys.modules, "huggingface_hub.errors", errors) + real_replace = readiness_module.os.replace + + def _fail_publish(source: object, destination: object) -> None: + if cast(Path, source).name == "cache": + raise OSError("publish interruption") + real_replace(cast(Path, source), cast(Path, destination)) + + monkeypatch.setattr(readiness_module.os, "replace", _fail_publish) + monkeypatch.setenv("SER_TRAINING_REPAIR_ALLOW_NETWORK", "1") + result = readiness_module._repair_pinned_model(settings) + assert result.succeeded is False + assert sentinel.read_bytes() == b"preserve" + assert not list(live_cache.parent.glob(f".{live_cache.name}.rollback-*")) + + +def test_git_lfs_pointer_is_systematic_abort() -> None: + classified = classify_failure( + AudioIntegrityError("Audio file is an unmaterialized Git LFS pointer"), + scope=FailureScope.SAMPLE, + ) + assert classified.scope is FailureScope.CORPUS + assert classified.reason_code is FailureReasonCode.GIT_LFS_POINTER + assert classified.disposition is FailureDisposition.ABORT + + +def test_missing_file_requires_exact_contained_sample_provenance(tmp_path: Path) -> None: + sample = _utterance(tmp_path, "missing", "calm", value=0.1) + sample.audio_path.unlink() + unproven = classify_failure(FileNotFoundError("cache missing"), scope=FailureScope.SAMPLE) + wrong_path = classify_failure( + FileNotFoundError(2, "missing", str(tmp_path / "cache.bin")), + scope=FailureScope.SAMPLE, + sample=sample, + allowed_roots=(tmp_path,), + ) + proven = classify_failure( + FileNotFoundError(2, "missing", str(sample.audio_path)), + scope=FailureScope.SAMPLE, + sample=sample, + allowed_roots=(tmp_path,), + ) + assert unproven.disposition is FailureDisposition.ABORT + assert wrong_path.disposition is FailureDisposition.ABORT + assert proven.disposition is FailureDisposition.QUARANTINE + + +def test_quarantine_identity_ignores_audit_timestamps(tmp_path: Path) -> None: + sample = _inventory(tmp_path)[0] + classification = classify_failure( + AudioDecodeError("isolated decoder failure"), scope=FailureScope.SAMPLE + ) + first = build_quarantine_record( + sample=sample, + classification=classification, + occurred_at="2026-01-01T00:00:00+00:00", + retry_count=0, + ) + second = replace( + first, + first_occurrence="2026-02-01T00:00:00+00:00", + last_occurrence="2026-02-01T00:00:01+00:00", + ) + first_path = tmp_path / "first.jsonl" + second_path = tmp_path / "second.jsonl" + assert quarantine_ledger_digest([first]) == quarantine_ledger_digest([second]) + assert write_quarantine_ledger(first_path, [first]) == write_quarantine_ledger( + second_path, [second] + ) + assert first_path.read_text() != second_path.read_text() + + +def test_unknown_sample_exception_aborts_with_original_diagnostic() -> None: + classified = classify_failure( + ValueError("non-finite backend output"), scope=FailureScope.SAMPLE + ) + assert classified.disposition is FailureDisposition.ABORT + assert classified.diagnostic == "non-finite backend output" + + +def test_typed_non_sample_containment_actions_are_allowlisted_only() -> None: + assert ( + classify_failure( + WindowContainmentError("low variance"), scope=FailureScope.WINDOW + ).disposition + is FailureDisposition.CONTINUE + ) + assert ( + classify_failure( + CacheEntryCorruptError("derived cache"), scope=FailureScope.CACHE + ).disposition + is FailureDisposition.RECOMPUTE + ) + assert ( + classify_failure( + OptionalArtifactError("advisory export"), scope=FailureScope.OPTIONAL_ARTIFACT + ).disposition + is FailureDisposition.CONTINUE + ) + for scope in ( + FailureScope.WINDOW, + FailureScope.CACHE, + FailureScope.OPTIONAL_ARTIFACT, + ): + assert classify_failure(RuntimeError("unknown"), scope=scope).disposition is ( + FailureDisposition.ABORT + ) + + +def test_shared_readiness_denies_restricted_backend_before_smoke( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + import ser._internal.license_check as license_check # noqa: TID251 + + settings = replace( + _settings(tmp_path), + runtime_flags=RuntimeFlags( + profile_pipeline=True, + accurate_research_profile=True, + restricted_backends=False, + ), + ) + monkeypatch.setattr(license_check, "parse_allowed_restricted_backends_env", frozenset) + monkeypatch.setattr(license_check, "load_persisted_backend_consents", lambda **_kwargs: {}) + smoke_called = False + + def _smoke(**_kwargs: object) -> SmokeResult: + nonlocal smoke_called + smoke_called = True + raise AssertionError("denied backend must not be constructed") + + report, _ = run_training_readiness( + settings=settings, + load_utterances=lambda: _inventory(tmp_path), + smoke_runner=_smoke, + ) + assert smoke_called is False + assert any(finding.check == "restricted_backend_access" for finding in report.findings) + + +@pytest.mark.parametrize( + ("policy_override", "message"), + [ + ({"max_absolute": 0}, "Absolute"), + ({"max_global_ratio": 0.124}, "Global"), + ({"max_corpus_ratio": 0.124}, "corpus"), + ({"max_class_ratio": 0.249}, "class"), + ({"max_per_reason": 0}, "reason"), + ({"strict": True}, "Strict"), + ({"min_remaining_per_class_split": 4}, "remaining"), + ], +) +def test_every_quarantine_budget_boundary_aborts( + tmp_path: Path, + policy_override: dict[str, object], + message: str, +) -> None: + samples = _inventory(tmp_path) + base = QuarantinePolicy( + max_absolute=4, + max_global_ratio=0.5, + max_corpus_ratio=0.5, + max_class_ratio=0.5, + max_per_reason=2, + min_remaining_per_class_split=1, + ) + policy = QuarantinePolicy( + max_absolute=cast(int, policy_override.get("max_absolute", base.max_absolute)), + max_global_ratio=cast( + float, policy_override.get("max_global_ratio", base.max_global_ratio) + ), + max_corpus_ratio=cast( + float, policy_override.get("max_corpus_ratio", base.max_corpus_ratio) + ), + max_class_ratio=cast(float, policy_override.get("max_class_ratio", base.max_class_ratio)), + max_per_reason=cast(int, policy_override.get("max_per_reason", base.max_per_reason)), + min_remaining_per_class_split=cast( + int, + policy_override.get( + "min_remaining_per_class_split", base.min_remaining_per_class_split + ), + ), + strict=cast(bool, policy_override.get("strict", base.strict)), + ) + classification = classify_failure( + AudioDecodeError("isolated decode"), scope=FailureScope.SAMPLE + ) + with pytest.raises(QuarantineBudgetExceeded, match=message): + enforce_quarantine_budget( + policy=policy, + all_samples=samples, + existing_records=[], + candidate=samples[0], + classification=classification, + ) + + +def test_allowlisted_sample_failure_at_exact_budget_is_accepted(tmp_path: Path) -> None: + samples = _inventory(tmp_path) + classification = classify_failure( + AudioDecodeError("isolated decode"), scope=FailureScope.SAMPLE + ) + enforce_quarantine_budget( + policy=QuarantinePolicy( + max_absolute=1, + max_global_ratio=0.25, + max_corpus_ratio=0.25, + max_class_ratio=0.5, + max_per_reason=1, + min_remaining_per_class_split=1, + ), + all_samples=samples, + existing_records=[], + candidate=samples[0], + classification=classification, + ) + record = build_quarantine_record( + sample=samples[0], + classification=classification, + occurred_at="2026-01-01T00:00:00+00:00", + retry_count=0, + ) + assert record.path_digest != str(samples[0].audio_path) + assert record.disposition is FailureDisposition.QUARANTINE + + +@given( + total=st.integers(min_value=2, max_value=200), + ratio=st.floats(min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False), +) +def test_global_budget_arithmetic_matches_exact_projected_ratio( + total: int, + ratio: float, +) -> None: + samples = [ + Utterance(1, f"sample-{index}", "fixture", Path(f"{index}.wav"), "calm") + for index in range(total) + ] + classification = classify_failure( + AudioDecodeError("isolated decoder failure"), + scope=FailureScope.SAMPLE, + ) + policy = QuarantinePolicy(total, ratio, 1.0, 1.0, total, 0) + if 1 / total <= ratio: + enforce_quarantine_budget( + policy=policy, + all_samples=samples, + existing_records=[], + candidate=samples[0], + classification=classification, + ) + else: + with pytest.raises(QuarantineBudgetExceeded, match="Global"): + enforce_quarantine_budget( + policy=policy, + all_samples=samples, + existing_records=[], + candidate=samples[0], + classification=classification, + ) + + +@given(scope=st.sampled_from(list(FailureScope))) +def test_unknown_and_unproven_missing_failures_never_become_exclusions( + scope: FailureScope, +) -> None: + unknown = classify_failure(RuntimeError("systemic"), scope=scope) + missing = classify_failure(FileNotFoundError("unproven"), scope=scope) + assert unknown.disposition is FailureDisposition.ABORT + assert missing.disposition is FailureDisposition.ABORT + + +def test_smoke_selection_remains_linear_and_capped() -> None: + samples = [ + Utterance(1, f"sample-{index:05d}", "fixture", Path(f"{index}.wav"), "calm") + for index in range(10_000) + ] + started = datetime.now(UTC) + selected = select_smoke_samples(samples, cap=16) + elapsed = (datetime.now(UTC) - started).total_seconds() + assert len(selected) == 16 + assert elapsed < 1.0 + + +def _readiness(settings: AppConfig) -> ReadinessReport: + return ReadinessReport( + schema_version=1, + created_at="2026-01-01T00:00:00+00:00", + profile="fast", + settings_digest=digest_payload(settings), + registry_digest="registry", + manifest_digest="manifest", + media_digest="media", + selected_sample_ids=("calm-1",), + findings=(), + ) + + +def _build_test_plan( + settings: AppConfig, + payload: Path, + *, + cache_keys: tuple[str, ...] = ("key",), + created_at: str | None = None, +) -> PreparedPlan: + return build_prepared_plan( + settings=settings, + readiness=_readiness(settings), + backend_id="handcrafted", + model_id="builtin", + model_revision="builtin", + device="cpu", + dtype="float64", + recipe_digest="recipe", + split_ledger_digest="split", + quarantine_ledger_digest="quarantine", + cache_namespace="features", + cache_version="v1", + cache_keys=cache_keys, + effective_counts={"class": {"calm": 2, "happy": 2}}, + feature_shape=(4, 3), + feature_dtype="float64", + windowing_policy={"strategy": "utterance"}, + noise_statistics={}, + payload_path=payload, + package_version="1.0.0", + created_at=created_at, + ) + + +def test_prepared_plan_is_deterministic_atomic_and_digest_verified(tmp_path: Path) -> None: + settings = _settings(tmp_path) + payload = tmp_path / "features.npz" + payload.write_bytes(b"safe-features") + first = _build_test_plan( + settings, + payload, + cache_keys=("b", "a"), + created_at="2026-01-01T00:00:00+00:00", + ) + second = _build_test_plan( + settings, + payload, + cache_keys=("b", "a"), + created_at="2026-01-01T00:00:00+00:00", + ) + path = tmp_path / "plan.json" + assert first == second + assert first.cache_keys == ("a", "b") + write_prepared_plan(path, first) + assert load_prepared_plan(path) == first + + raw = path.read_text(encoding="utf-8").replace('"device": "cpu"', '"device": "cuda"') + path.write_text(raw, encoding="utf-8") + with pytest.raises(PreparedPlanError, match="digest mismatch"): + load_prepared_plan(path) + + +@pytest.mark.parametrize( + ("field", "invalid"), + [ + ("cache_keys", [1]), + ("feature_shape", [4, True]), + ("effective_counts", {"class": {"calm": -1}}), + ("sample_ledger", ["not-an-object"]), + ("disposition_counts", {"included": 1.5}), + ], +) +def test_redigested_malformed_plan_is_rejected_at_typed_boundary( + tmp_path: Path, + field: str, + invalid: object, +) -> None: + settings = _settings(tmp_path) + payload = tmp_path / "features.npz" + payload.write_bytes(b"safe-features") + plan = _build_test_plan(settings, payload) + raw = plan.to_dict() + raw[field] = invalid + unsigned = dict(raw) + unsigned.pop("overall_digest") + raw["overall_digest"] = digest_payload(unsigned) + path = tmp_path / "malformed-plan.json" + atomic_write_json(path, raw) + with pytest.raises(PreparedPlanError, match=field): + load_prepared_plan(path) + + +def test_interrupted_atomic_publication_leaves_no_ready_document( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + destination = tmp_path / "prepared.json" + + def _fail_replace(_source: object, _destination: object) -> None: + raise OSError("simulated interruption") + + monkeypatch.setattr("ser._internal.models.training_readiness.os.replace", _fail_replace) + with pytest.raises(OSError, match="simulated interruption"): + atomic_write_json(destination, {"ready": True}) + assert not destination.exists() + assert not list(tmp_path.glob(f".{destination.name}.*")) + + +@pytest.mark.parametrize( + "field", + [ + "settings_digest", + "registry_digest", + "manifest_digest", + "media_digest", + "recipe_digest", + "split_ledger_digest", + "quarantine_ledger_digest", + "backend_id", + "model_id", + "model_revision", + "device", + "dtype", + "cache_namespace", + "cache_version", + "cache_keys", + "package_version", + ], +) +def test_any_relevant_input_change_invalidates_prepared_plan( + tmp_path: Path, + field: str, +) -> None: + settings = _settings(tmp_path) + payload = tmp_path / "features.npz" + payload.write_bytes(b"safe-features") + readiness = _readiness(settings) + plan = _build_test_plan(settings, payload) + changed_readiness = replace( + readiness, + settings_digest="changed" if field == "settings_digest" else readiness.settings_digest, + registry_digest="changed" if field == "registry_digest" else readiness.registry_digest, + manifest_digest="changed" if field == "manifest_digest" else readiness.manifest_digest, + media_digest="changed" if field == "media_digest" else readiness.media_digest, + ) + with pytest.raises(PreparedPlanError, match=field): + validate_prepared_plan( + plan, + settings=settings, + readiness=changed_readiness, + backend_id="changed" if field == "backend_id" else "handcrafted", + model_id="changed" if field == "model_id" else "builtin", + model_revision="changed" if field == "model_revision" else "builtin", + device="changed" if field == "device" else "cpu", + dtype="changed" if field == "dtype" else "float64", + recipe_digest="changed" if field == "recipe_digest" else "recipe", + split_ledger_digest="changed" if field == "split_ledger_digest" else "split", + quarantine_ledger_digest=( + "changed" if field == "quarantine_ledger_digest" else "quarantine" + ), + cache_namespace="changed" if field == "cache_namespace" else "features", + cache_version="changed" if field == "cache_version" else "v1", + cache_keys=("changed",) if field == "cache_keys" else ("key",), + package_version="changed" if field == "package_version" else "1.0.0", + ) + + +def test_settings_digest_serializes_read_only_mapping_config_fields() -> None: + """Regression: settings_digest must not choke on MappingProxyType config fields. + + AppConfig.emotions is exposed as a read-only MappingProxyType. The digest path + previously serialized settings via dataclasses.asdict(), whose deep-copy raised + 'TypeError: cannot pickle mappingproxy object'. That crash only surfaced once the + readiness pass stopped stalling in split_feasibility and reached report assembly. + """ + # reload_settings() stores emotions as a read-only MappingProxyType; construct it + # explicitly so the regression does not depend on ambient configuration. + settings = AppConfig(emotions=MappingProxyType({"01": "calm", "02": "happy"})) + assert isinstance(settings.emotions, MappingProxyType) # the crash trigger + + digest = settings_digest(settings) + assert isinstance(digest, str) and len(digest) == 64 + assert digest == settings_digest(settings) # deterministic + + # The read-only mapping's contents are actually captured in the canonical payload. + payload = canonical_json_bytes(settings) + assert b'"calm"' in payload and b'"happy"' in payload diff --git a/tests/suites/unit/models/test_training_readiness_evidence.py b/tests/suites/unit/models/test_training_readiness_evidence.py new file mode 100644 index 0000000..e75cdca --- /dev/null +++ b/tests/suites/unit/models/test_training_readiness_evidence.py @@ -0,0 +1,430 @@ +"""Executable evidence that readiness failures cannot reach model fitting.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from typing import cast + +import numpy as np +import pytest +import soundfile as sf +from hypothesis import given +from hypothesis import strategies as st + +from ser._internal.config.schema import ( # noqa: TID251 + DataLoaderConfig, + DatasetConfig, + ModelsConfig, +) +from ser._internal.data.manifest import SplitName, Utterance # noqa: TID251 +from ser._internal.models import training_readiness as readiness # noqa: TID251 +from ser._internal.models.training_readiness import ( # noqa: TID251 + CacheEntryCorruptError, + FailureDisposition, + FailureReasonCode, + FailureScope, + OptionalArtifactError, + QuarantineBudgetExceeded, + QuarantinePolicy, + SmokeResult, + TrainingReadinessError, + WindowContainmentError, + build_quarantine_record, + classify_failure, + enforce_quarantine_budget, + ensure_training_ready, + normalized_pcm_digest, +) +from ser._internal.utils.audio_utils import AudioDecodeError # noqa: TID251 +from ser.config import AppConfig + + +def _settings(tmp_path: Path) -> AppConfig: + """Builds an isolated readiness configuration with explicit budgets.""" + return AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + data_loader=DataLoaderConfig( + max_workers=1, + max_failed_file_ratio=0.5, + max_failed_files=4, + max_failed_file_ratio_per_corpus=0.5, + max_failed_file_ratio_per_class=0.5, + max_failures_per_reason=2, + min_remaining_per_class_split=1, + ), + models=ModelsConfig( + folder=tmp_path / "models", + model_cache_dir=tmp_path / "cache", + ), + ) + + +def _utterance( + tmp_path: Path, + sample_id: str, + label: str, + *, + value: float, + split: SplitName | None = None, +) -> Utterance: + """Writes one deterministic tiny WAV and returns its manifest row.""" + path = tmp_path / f"{sample_id}.wav" + sf.write(path, np.full(80, value, dtype=np.float32), 8_000, subtype="PCM_16") + return Utterance( + schema_version=1, + sample_id=sample_id, + corpus="fixture", + audio_path=path, + label=label, + language="en", + split=split, + ) + + +def _inventory(tmp_path: Path) -> list[Utterance]: + """Returns a minimal feasible unsplit two-class inventory.""" + return [ + _utterance(tmp_path, f"calm-{index}", "calm", value=0.10 + index / 100) + for index in range(4) + ] + [ + _utterance(tmp_path, f"happy-{index}", "happy", value=0.20 + index / 100) + for index in range(4) + ] + + +def _native_inventory(tmp_path: Path) -> list[Utterance]: + """Returns complete train/dev/test partitions with both classes represented.""" + rows: list[Utterance] = [] + for partition_index, split in enumerate(("train", "dev", "test")): + typed_split = cast(SplitName, split) + rows.extend( + ( + _utterance( + tmp_path, + f"{split}-calm", + "calm", + value=0.10 + partition_index / 100, + split=typed_split, + ), + _utterance( + tmp_path, + f"{split}-happy", + "happy", + value=0.20 + partition_index / 100, + split=typed_split, + ), + ) + ) + return rows + + +class _ClassifierTrap: + """Records forbidden classifier construction and fit calls.""" + + def __init__(self, calls: list[str]) -> None: + calls.append("classifier") + self._calls = calls + + def fit(self, features: object, labels: object) -> None: + """Records a forbidden fit boundary.""" + del features, labels + self._calls.append("fit") + + +def _assert_blocked_before_fit( + *, + settings: AppConfig, + load_utterances: Callable[[], list[Utterance] | None], + expected_reason: FailureReasonCode, + smoke_error: Exception | None = None, + expected_smoke_calls: int = 0, +) -> None: + """Asserts mandatory readiness raises before classifier construction or fit.""" + smoke_calls: list[tuple[str, ...]] = [] + training_calls: list[str] = [] + + def _smoke( + *, + settings: AppConfig, + samples: Sequence[Utterance], + probe_cache_dir: Path, + ) -> SmokeResult: + del settings, probe_cache_dir + smoke_calls.append(tuple(item.sample_id for item in samples)) + if smoke_error is not None: + raise smoke_error + return SmokeResult( + attempted=len(samples), + succeeded=len(samples), + feature_dim=4, + cache_round_trip=True, + backend_id="stub", + model_id="stub-model", + device="cpu", + dtype="float32", + ) + + with pytest.raises(TrainingReadinessError) as captured: + ensure_training_ready( + settings=settings, + load_utterances=load_utterances, + smoke_runner=_smoke, + ) + classifier = _ClassifierTrap(training_calls) + classifier.fit(object(), object()) + + assert training_calls == [] + assert len(smoke_calls) == expected_smoke_calls + assert any( + finding.blocking and finding.reason_code is expected_reason + for finding in captured.value.report.findings + ) + + +@pytest.mark.parametrize( + ("case", "expected_reason"), + [ + pytest.param("backend", FailureReasonCode.BACKEND_UNAVAILABLE, id="missing-backend"), + pytest.param("model", FailureReasonCode.BACKEND_UNAVAILABLE, id="missing-model"), + pytest.param( + "numeric", FailureReasonCode.INVALID_CONFIGURATION, id="invalid-numeric-config" + ), + pytest.param("manifest", FailureReasonCode.MANIFEST_INVALID, id="invalid-manifest"), + pytest.param("media", FailureReasonCode.MEDIA_NOT_REGULAR, id="invalid-media"), + pytest.param("output", FailureReasonCode.OUTPUT_UNWRITABLE, id="unwritable-output"), + pytest.param("disk", FailureReasonCode.DISK_SPACE_LOW, id="low-disk"), + ], +) +def test_preflight_failures_block_before_classifier_and_fit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + case: str, + expected_reason: FailureReasonCode, +) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + + def _load_samples() -> list[Utterance]: + return samples + + loader: Callable[[], list[Utterance] | None] = _load_samples + smoke_error: Exception | None = None + expected_smoke_calls = 0 + + if case == "backend": + smoke_error = ImportError("feature backend is not installed") + expected_smoke_calls = 1 + elif case == "model": + smoke_error = RuntimeError("model load failed: pinned snapshot is missing") + expected_smoke_calls = 1 + elif case == "numeric": + settings = replace( + settings, + training=replace(settings.training, test_size=float("nan")), + ) + elif case == "manifest": + + def _invalid_manifest() -> list[Utterance]: + raise ValueError("manifest record lacks sample_id") + + loader = _invalid_manifest + elif case == "media": + media_directory = tmp_path / "not-a-file" + media_directory.mkdir() + samples[0] = replace(samples[0], audio_path=media_directory) + elif case == "output": + + def _deny_output(_path: Path) -> None: + raise PermissionError("output is read-only") + + monkeypatch.setattr(readiness, "_probe_directory", _deny_output) # noqa: SLF001 + elif case == "disk": + monkeypatch.setattr( + readiness.shutil, + "disk_usage", + lambda _path: SimpleNamespace(total=1, used=1, free=0), + ) + else: + raise AssertionError(f"Unhandled test case: {case}") + + _assert_blocked_before_fit( + settings=settings, + load_utterances=loader, + expected_reason=expected_reason, + smoke_error=smoke_error, + expected_smoke_calls=expected_smoke_calls, + ) + + +@pytest.mark.parametrize("identity", ["content", "speaker", "session"]) +def test_protected_identity_leakage_blocks_before_classifier_and_fit( + tmp_path: Path, + identity: str, +) -> None: + settings = _settings(tmp_path) + samples = _native_inventory(tmp_path) + + if identity == "content": + duplicated_pcm = np.full(80, 0.10, dtype=np.float32) + sf.write(samples[2].audio_path, duplicated_pcm, 8_000, subtype="PCM_16") + digest = normalized_pcm_digest(samples[0].audio_path) + samples[0] = replace(samples[0], normalized_audio_sha256=digest) + samples[2] = replace(samples[2], normalized_audio_sha256=digest) + elif identity == "speaker": + samples[0] = replace(samples[0], speaker_id="fixture:speaker-leak") + samples[2] = replace(samples[2], speaker_id="fixture:speaker-leak") + elif identity == "session": + samples[0] = replace(samples[0], session_id="fixture:session-leak") + samples[2] = replace(samples[2], session_id="fixture:session-leak") + else: + raise AssertionError(f"Unhandled identity: {identity}") + + _assert_blocked_before_fit( + settings=settings, + load_utterances=lambda: samples, + expected_reason=( + FailureReasonCode.DUPLICATE_CONTENT + if identity == "content" + else FailureReasonCode.SPLIT_LEAKAGE + ), + ) + + +@pytest.mark.parametrize("case", ["class", "split"]) +def test_insufficient_class_or_split_support_blocks_before_fit( + tmp_path: Path, + case: str, +) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path)[:4] + if case == "split": + samples = _native_inventory(tmp_path) + settings = replace( + settings, + data_loader=replace( + settings.data_loader, + min_remaining_per_class_split=2, + ), + ) + + _assert_blocked_before_fit( + settings=settings, + load_utterances=lambda: samples, + expected_reason=FailureReasonCode.INSUFFICIENT_CLASS_SUPPORT, + ) + + +@given( + scope=st.sampled_from(list(FailureScope)), + failure_kind=st.sampled_from(("unknown", "window", "cache", "optional", "decode", "timeout")), +) +def test_failure_dispositions_are_confined_to_exact_typed_scopes( + scope: FailureScope, + failure_kind: str, +) -> None: + error_by_kind: dict[str, Exception] = { + "unknown": RuntimeError("systemic"), + "window": WindowContainmentError("low variance"), + "cache": CacheEntryCorruptError("checksum mismatch"), + "optional": OptionalArtifactError("optional export failed"), + "decode": AudioDecodeError("isolated decode failure"), + "timeout": TimeoutError("transient local read"), + } + expected_by_kind: dict[str, tuple[FailureScope, FailureDisposition]] = { + "window": (FailureScope.WINDOW, FailureDisposition.CONTINUE), + "cache": (FailureScope.CACHE, FailureDisposition.RECOMPUTE), + "optional": (FailureScope.OPTIONAL_ARTIFACT, FailureDisposition.CONTINUE), + "decode": (FailureScope.SAMPLE, FailureDisposition.QUARANTINE), + "timeout": (FailureScope.SAMPLE, FailureDisposition.BOUNDED_RETRY), + } + + classified = classify_failure(error_by_kind[failure_kind], scope=scope) + expected = expected_by_kind.get(failure_kind) + if expected is not None and scope is expected[0]: + assert classified.disposition is expected[1] + else: + assert classified.disposition is FailureDisposition.ABORT + if classified.disposition is FailureDisposition.QUARANTINE: + assert classified.scope is FailureScope.SAMPLE + + +@given( + total=st.integers(min_value=1, max_value=50), + raw_existing=st.integers(min_value=0, max_value=100), + max_absolute=st.integers(min_value=0, max_value=50), + global_ratio=st.floats(0.0, 1.0, allow_nan=False, allow_infinity=False), + corpus_ratio=st.floats(0.0, 1.0, allow_nan=False, allow_infinity=False), + class_ratio=st.floats(0.0, 1.0, allow_nan=False, allow_infinity=False), + max_per_reason=st.integers(min_value=0, max_value=50), + min_remaining=st.integers(min_value=0, max_value=50), + strict=st.booleans(), +) +def test_quarantine_budget_accepts_exactly_the_conjunction_of_all_limits( + total: int, + raw_existing: int, + max_absolute: int, + global_ratio: float, + corpus_ratio: float, + class_ratio: float, + max_per_reason: int, + min_remaining: int, + strict: bool, +) -> None: + samples = [ + Utterance(1, f"sample-{index}", "fixture", Path(f"{index}.wav"), "calm") + for index in range(total) + ] + existing_count = raw_existing % total + classification = classify_failure( + AudioDecodeError("isolated decode failure"), + scope=FailureScope.SAMPLE, + ) + records = [ + build_quarantine_record( + sample=sample, + classification=classification, + occurred_at="2026-01-01T00:00:00+00:00", + retry_count=0, + ) + for sample in samples[:existing_count] + ] + policy = QuarantinePolicy( + max_absolute=max_absolute, + max_global_ratio=global_ratio, + max_corpus_ratio=corpus_ratio, + max_class_ratio=class_ratio, + max_per_reason=max_per_reason, + min_remaining_per_class_split=min_remaining, + strict=strict, + ) + projected = existing_count + 1 + projected_ratio = projected / total + should_accept = ( + not strict + and projected <= max_absolute + and projected_ratio <= global_ratio + and projected_ratio <= corpus_ratio + and projected_ratio <= class_ratio + and projected <= max_per_reason + and total - projected >= min_remaining + ) + + def _enforce() -> None: + enforce_quarantine_budget( + policy=policy, + all_samples=samples, + existing_records=records, + candidate=samples[existing_count], + classification=classification, + ) + + if should_accept: + _enforce() + else: + with pytest.raises(QuarantineBudgetExceeded): + _enforce() From 91c3e65a38f73d11bc2ee722bc44dd7e15bac938 Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:19:01 -0300 Subject: [PATCH 07/10] feat(training): add preparation and prepared-plan orchestration Routes every training entrypoint through one readiness contract, then lets the expensive part be done once and reused. Preparation writes a non-pickle NPZ feature payload and publishes a plan only after the payload lands, so an interrupted run never leaves a plan pointing at a partial payload. A prepared plan is only accepted when it still describes reality: canonical plan and payload digests, profile and settings, backend and model revision, device and dtype, registry, manifests, media fingerprints, recipe, split and quarantine ledgers, cache namespace, and the actual sample and window ledgers. Backend plans require the exact commit resolved by the checked backend, so a mutable tag cannot mint a reusable plan. Any mismatch is a validation failure rather than a silently different run. Also wires dataset recipe routing into utterance loading and reports bounded progress with a credible ETA during preparation. --- ser/_internal/data/data_loader.py | 181 ++- .../models/accurate_feature_dataset.py | 10 +- .../models/accurate_training_preparation.py | 352 ++++- ser/_internal/models/artifact_persistence.py | 71 +- ser/_internal/models/fast_training.py | 89 +- .../models/feature_runtime_encoding.py | 19 +- .../models/medium_feature_dataset.py | 10 +- .../models/medium_training_preparation.py | 242 ++- .../models/profile_training_preparation.py | 58 +- ser/_internal/models/training_entrypoints.py | 121 +- ser/_internal/models/training_execution.py | 15 +- .../models/training_orchestration.py | 1400 +++++++++++++++++ ser/_internal/models/training_preparation.py | 65 +- ser/_internal/models/training_types.py | 2 + .../models/test_emotion_model_contracts.py | 71 +- ...test_training_preparation_and_execution.py | 10 + tests/suites/unit/data/test_data_loader.py | 52 + .../models/test_accurate_training_artifact.py | 105 +- .../test_accurate_training_preparation.py | 1 + .../unit/models/test_artifact_persistence.py | 182 ++- .../test_medium_training_preparation.py | 1 + .../test_training_mode_orchestration.py | 1036 ++++++++++++ .../test_training_resilience_evidence.py | 180 +++ .../models/test_training_smoke_performance.py | 256 +++ 24 files changed, 4326 insertions(+), 203 deletions(-) create mode 100644 ser/_internal/models/training_orchestration.py create mode 100644 tests/suites/unit/models/test_training_mode_orchestration.py create mode 100644 tests/suites/unit/models/test_training_resilience_evidence.py create mode 100644 tests/suites/unit/models/test_training_smoke_performance.py diff --git a/ser/_internal/data/data_loader.py b/ser/_internal/data/data_loader.py index 9588c48..b0ad0e7 100644 --- a/ser/_internal/data/data_loader.py +++ b/ser/_internal/data/data_loader.py @@ -4,22 +4,31 @@ import logging import multiprocessing as mp import os -from collections.abc import Collection, Mapping +import time +from collections.abc import Callable, Collection, Mapping, Sequence +from dataclasses import replace from functools import partial from pathlib import Path -from typing import Any, NamedTuple +from typing import Any, NamedTuple, cast import numpy as np from numpy.typing import NDArray from sklearn.model_selection import train_test_split from ser._internal.data.adapters.ravdess import build_ravdess_utterances -from ser._internal.data.dataset_prepare import prepare_from_registry_entry +from ser._internal.data.dataset_audit import audit_dataset_recipe +from ser._internal.data.dataset_prepare import ( + prepare_from_registry_entry, + validate_registered_dataset_integrity, +) from ser._internal.data.dataset_registry import load_dataset_registry from ser._internal.data.label_ontology import resolve_label_ontology -from ser._internal.data.manifest import Utterance +from ser._internal.data.manifest import SplitName, Utterance from ser._internal.data.manifest_jsonl import load_manifest_jsonl +from ser._internal.data.recipe import load_dataset_recipe from ser._internal.features.feature_extractor import _extract_feature_for_settings +from ser._internal.repr import HandcraftedBackend +from ser._internal.utils.audio_utils import read_audio_file from ser._internal.utils.logger import get_logger from ser.config import AppConfig, AudioReadConfig, FeatureFlags, reload_settings @@ -37,9 +46,17 @@ def _resolve_boundary_settings(settings: AppConfig | None) -> AppConfig: return settings if settings is not None else reload_settings() -def _load_utterances_for_settings(settings: AppConfig) -> list[Utterance] | None: +def _load_utterances_for_settings( + settings: AppConfig, + *, + allow_prepare: bool = True, +) -> list[Utterance] | None: """Loads utterances using one explicit settings snapshot.""" ontology = resolve_label_ontology(settings) + dataset_recipe = getattr(settings.dataset, "recipe", None) + strict_dataset_audit = bool(getattr(settings.dataset, "strict_audit", False)) + if strict_dataset_audit and not isinstance(dataset_recipe, str): + raise RuntimeError("Strict dataset audit requires an explicit versioned dataset recipe.") def _validate_utterances(utterances: list[Utterance]) -> list[Utterance] | None: if not utterances: @@ -55,7 +72,34 @@ def _validate_utterances(utterances: list[Utterance]) -> list[Utterance] | None: raise RuntimeError( "Duplicate sample_id values across manifests: " + ", ".join(sorted(duplicates)) ) - labels = [utterance.label for utterance in utterances] + if isinstance(dataset_recipe, str): + recipe = load_dataset_recipe(dataset_recipe) + report = audit_dataset_recipe( + utterances, + recipe=recipe, + seed=settings.training.random_state, + strict=strict_dataset_audit, + ) + by_sample_id = {utterance.sample_id: utterance for utterance in utterances} + utterances = [ + replace(by_sample_id[entry.sample_id], split=cast(SplitName, entry.split)) + for entry in report.ledger + if "primary_emotion" in entry.tasks + and entry.split in {"train", "dev", "test"} + and by_sample_id[entry.sample_id].label is not None + ] + logger.info( + "Dataset audit passed (recipe=%s@%s recipe_digest=%s ledger_digest=%s counters=%s).", + report.recipe_id, + report.recipe_revision, + report.recipe_digest, + report.split_ledger_digest, + report.counters, + ) + if not utterances: + logger.warning("Dataset recipe produced zero primary-emotion training rows.") + return None + labels = [utterance.label for utterance in utterances if utterance.label is not None] if len(set(labels)) < 2: logger.warning("At least two emotion classes are required to train the model.") return None @@ -66,17 +110,31 @@ def _load_manifest_paths( *, base_dirs: Mapping[Path, Path] | None = None, ) -> list[Utterance] | None: + started_at = time.perf_counter() + logger.info("MANIFEST_LOAD_START manifests=%d", len(manifest_paths)) utterances: list[Utterance] = [] for manifest_path in manifest_paths: + manifest_started_at = time.perf_counter() resolved_manifest_path = manifest_path.expanduser() base_dir = base_dirs.get(resolved_manifest_path) if base_dirs is not None else None - utterances.extend( - load_manifest_jsonl( - resolved_manifest_path, - ontology=ontology, - base_dir=base_dir, - ) + manifest_utterances = load_manifest_jsonl( + resolved_manifest_path, + ontology=ontology, + base_dir=base_dir, ) + utterances.extend(manifest_utterances) + logger.info( + "MANIFEST_LOAD_DONE path=%s rows=%d elapsed=%.1fs", + resolved_manifest_path, + len(manifest_utterances), + time.perf_counter() - manifest_started_at, + ) + logger.info( + "MANIFEST_LOAD_DONE total_rows=%d manifests=%d elapsed=%.1fs", + len(utterances), + len(manifest_paths), + time.perf_counter() - started_at, + ) return _validate_utterances(utterances) if settings.dataset.manifest_paths: @@ -84,15 +142,22 @@ def _load_manifest_paths( registry = load_dataset_registry(settings=settings) if registry: + logger.info("DATASET_REGISTRY_LOAD datasets=%d", len(registry)) manifest_paths: list[Path] = [] base_dirs: dict[Path, Path] = {} for entry in registry.values(): manifest_path = entry.manifest_path.expanduser() dataset_root = entry.dataset_root.expanduser() + validate_registered_dataset_integrity(entry) if manifest_path.is_file(): manifest_paths.append(manifest_path) base_dirs[manifest_path] = dataset_root continue + if not allow_prepare: + raise RuntimeError( + f"Registered dataset {entry.dataset_id!r} is missing its manifest at " + f"{manifest_path}; run prepare-only with --repair to rebuild it." + ) try: built_paths = prepare_from_registry_entry( settings=settings, @@ -120,7 +185,7 @@ def _load_manifest_paths( ) return _load_manifest_paths(deduped_paths, base_dirs=base_dirs) - return build_ravdess_utterances( + fallback = build_ravdess_utterances( dataset_root=settings.dataset.folder, dataset_glob_pattern=settings.dataset.glob_pattern, emotion_code_map=dict(settings.emotions), @@ -128,11 +193,19 @@ def _load_manifest_paths( ontology=ontology, max_failed_file_ratio=settings.data_loader.max_failed_file_ratio, ) + return _validate_utterances(fallback) if fallback is not None else None -def load_utterances(*, settings: AppConfig | None = None) -> list[Utterance] | None: +def load_utterances( + *, + settings: AppConfig | None = None, + allow_prepare: bool = True, +) -> list[Utterance] | None: """Loads manifest utterances when configured, otherwise defaults to RAVDESS discovery.""" - return _load_utterances_for_settings(_resolve_boundary_settings(settings)) + return _load_utterances_for_settings( + _resolve_boundary_settings(settings), + allow_prepare=allow_prepare, + ) class ProcessFileResult(NamedTuple): @@ -195,8 +268,7 @@ def process_file( return ProcessFileResult( sample=None, error=( - "Skipping file with unexpected name format " - f"(missing emotion code): {file_name}" + f"Skipping file with unexpected name format (missing emotion code): {file_name}" ), ) @@ -234,7 +306,9 @@ def _load_labeled_audio_paths_for_settings( return None samples: list[LabeledAudioSample] = [ - (str(utterance.audio_path), utterance.label) for utterance in utterances + (str(utterance.audio_path), label) + for utterance in utterances + if (label := utterance.label) is not None ] labels: list[str] = [label for _, label in samples] if len(set(labels)) < 2: @@ -388,3 +462,74 @@ def load_data( test_size, settings=_resolve_boundary_settings(settings), ) + + +def load_checked_fast_data( + *, + utterances: Sequence[Utterance], + settings: AppConfig, + handle_sample_failure: Callable[[Utterance, Exception], bool] | None = None, +) -> DataSplit | None: + """Extracts fast features from exactly the readiness-checked utterance view.""" + from ser._internal.models.dataset_splitting import split_utterances + + if not utterances: + return None + train_utterances, test_utterances, _ = split_utterances( + samples=list(utterances), + settings=settings, + logger=logger, + ) + backend = HandcraftedBackend(feature_flags=settings.feature_flags) + + def _extract_partition( + partition: Sequence[Utterance], + ) -> tuple[FeatureMatrix, LabelList]: + from ser._internal.models.training_orchestration import ( # noqa: TID251 + record_preparation_progress, + ) + + rows: list[FeatureVector] = [] + labels: LabelList = [] + total = len(partition) + for processed, utterance in enumerate(partition, start=1): + try: + audio, sample_rate = read_audio_file( + str(utterance.audio_path), + start_seconds=utterance.start_seconds, + duration_seconds=utterance.duration_seconds, + audio_read_config=settings.audio_read, + ) + feature = np.asarray( + backend.extract_vector(audio=audio, sample_rate=sample_rate), + dtype=np.float64, + ) + except Exception as error: + if handle_sample_failure is not None and handle_sample_failure(utterance, error): + record_preparation_progress( + processed=processed, + total=total, + sample_id=utterance.sample_id, + ) + continue + raise + if feature.ndim != 1 or feature.size <= 0 or not np.all(np.isfinite(feature)): + raise ValueError( + f"Fast feature contract failed for sample {utterance.sample_id!r}." + ) + rows.append(feature) + labels.append(utterance.require_label()) + record_preparation_progress( + processed=processed, + total=total, + sample_id=utterance.sample_id, + ) + if not rows: + raise RuntimeError("Fast checked preparation produced an empty split partition.") + return np.vstack(rows).astype(np.float64, copy=False), labels + + x_train, y_train = _extract_partition(train_utterances) + x_test, y_test = _extract_partition(test_utterances) + if len(set(y_train)) < 2: + raise RuntimeError("Fast checked preparation left fewer than two training classes.") + return x_train, x_test, y_train, y_test diff --git a/ser/_internal/models/accurate_feature_dataset.py b/ser/_internal/models/accurate_feature_dataset.py index 99811fa..2f20e87 100644 --- a/ser/_internal/models/accurate_feature_dataset.py +++ b/ser/_internal/models/accurate_feature_dataset.py @@ -5,11 +5,11 @@ import logging from collections.abc import Callable, Sequence from functools import partial -from typing import TypeVar +from typing import TypeVar, cast import numpy as np -from ser._internal.data import EmbeddingCache +from ser._internal.data import EmbeddingCache, Utterance from ser._internal.models.feature_runtime_encoding import ( encode_sequence_with_cache, resolve_accurate_runtime_config, @@ -25,6 +25,7 @@ from ser._internal.models.profile_training_preparation import ( build_accurate_feature_dataset as _build_prepared_accurate_feature_dataset, ) +from ser._internal.models.training_orchestration import handle_sample_encoding_failure from ser._internal.pool import mean_std_pool, temporal_pooling_windows from ser._internal.repr import EncodedSequence, FeatureBackend, PoolingWindow from ser._internal.utils.audio_utils import read_audio_file @@ -149,6 +150,11 @@ def _encode_utterance(utterance: _UtteranceT) -> EncodedSequence: build_pooling_windows=build_pooling_windows, pool_features=pool_features, window_meta_factory=window_meta_factory, + handle_sample_failure=lambda utterance, error: handle_sample_encoding_failure( + settings=settings, + sample=cast(Utterance, utterance), + error=error, + ), ) diff --git a/ser/_internal/models/accurate_training_preparation.py b/ser/_internal/models/accurate_training_preparation.py index 01ae0c6..f3a2580 100644 --- a/ser/_internal/models/accurate_training_preparation.py +++ b/ser/_internal/models/accurate_training_preparation.py @@ -4,8 +4,9 @@ import logging from collections.abc import Callable +from dataclasses import replace from pathlib import Path -from typing import Any, Protocol, TypeVar +from typing import Any, Protocol, TypeVar, cast import numpy as np @@ -13,11 +14,30 @@ from ser._internal.models.accurate_training_execution import ( PreparedAccurateTrainingRunner, ) -from ser._internal.models.dataset_splitting import MediumSplitMetadata +from ser._internal.models.dataset_splitting import ( + MediumSplitMetadata, + medium_split_metadata_from_mapping, +) +from ser._internal.models.profile_runtime import ( + ACCURATE_BACKEND_ID, + ACCURATE_RESEARCH_BACKEND_ID, +) +from ser._internal.models.training_orchestration import ( + canonical_train_partition, + current_training_state, + prepare_until_quarantine_stable, + publish_prepared_features, + read_prepared_feature_payload, + reuse_checked_backend, + training_meta_sample_ids, + validate_operation_plan, +) from ser._internal.models.training_preparation import ( prepare_accurate_training_features, prepare_accurate_training_payload, ) +from ser._internal.models.training_readiness import TrainingMode +from ser._internal.models.training_support import WindowMeta from ser._internal.models.training_types import ( AccurateTrainingPreparation, ) @@ -65,32 +85,128 @@ def train_accurate_whisper_profile_model( embedding_cache_name: str = "accurate_embeddings", ) -> None: """Runs accurate training from settings using delegated hooks.""" - utterances, prepared = prepare_accurate_whisper_training( + resolved_model_id = resolve_model_id_for_settings(settings) + resolved_device, resolved_dtype = resolve_runtime_selectors_for_settings(settings) + plan = validate_operation_plan( settings=settings, - logger=logger, - load_utterances_for_training=load_utterances_for_training, - ensure_dataset_consents_for_training=lambda training_utterances: ( - ensure_dataset_consents_for_training(utterances=training_utterances) - ), - split_utterances=split_utterances, - resolve_model_id=lambda: resolve_model_id_for_settings(settings), - resolve_runtime_selectors=lambda: resolve_runtime_selectors_for_settings(settings), - build_backend=lambda model_id, runtime_device, runtime_dtype: build_backend( - model_id, - runtime_device, - runtime_dtype, - settings, - ), - build_feature_dataset=lambda partition, backend, cache, model_id: ( - build_feature_dataset( - utterances=partition, - backend=backend, - cache=cache, - model_id=model_id, - ) - ), - embedding_cache_path=settings.tmp_folder / embedding_cache_name, + backend_id=ACCURATE_BACKEND_ID, + model_id=resolved_model_id, + device=resolved_device, + dtype=resolved_dtype, ) + if plan is not None: + payload = read_prepared_feature_payload(plan) + metadata = payload.metadata + by_id = {item.sample_id: item for item in current_training_state().utterances} + train_ids = cast(list[str], metadata.get("train_sample_ids", [])) + test_ids = cast(list[str], metadata.get("test_sample_ids", [])) + split_raw = cast(dict[str, object], metadata.get("split_metadata", {})) + test_meta_raw = cast(list[dict[str, str]], metadata.get("test_meta", [])) + train_meta_raw = cast(list[dict[str, str]], metadata.get("train_meta", [])) + utterances = list(current_training_state().utterances) + prepared = cast( + AccurateTrainingPreparation[Utterance, MediumSplitMetadata, _MetaT], + AccurateTrainingPreparation( + train_utterances=[by_id[item] for item in train_ids], + test_utterances=[by_id[item] for item in test_ids], + split_metadata=medium_split_metadata_from_mapping(split_raw), + model_id=resolved_model_id, + runtime_device=resolved_device, + runtime_dtype=resolved_dtype, + x_train=payload.x_train, + y_train=payload.y_train, + train_meta=[WindowMeta(**item) for item in train_meta_raw], + x_test=payload.x_test, + y_test=payload.y_test, + test_meta=[WindowMeta(**item) for item in test_meta_raw], + ), + ) + else: + utterances, prepared = prepare_until_quarantine_stable( + settings=settings, + prepare=lambda: prepare_accurate_whisper_training( + settings=settings, + logger=logger, + load_utterances_for_training=lambda: list(current_training_state().utterances), + ensure_dataset_consents_for_training=lambda training_utterances: ( + ensure_dataset_consents_for_training(utterances=training_utterances) + ), + split_utterances=split_utterances, + resolve_model_id=lambda: resolved_model_id, + resolve_runtime_selectors=lambda: (resolved_device, resolved_dtype), + build_backend=lambda model_id, runtime_device, runtime_dtype: cast( + WhisperBackend, + reuse_checked_backend( + backend_id=ACCURATE_BACKEND_ID, + model_id=model_id, + device=runtime_device, + dtype=runtime_dtype, + build=lambda: build_backend( + model_id, + runtime_device, + runtime_dtype, + settings, + ), + ), + ), + build_feature_dataset=lambda partition, backend, cache, model_id: ( + build_feature_dataset( + utterances=partition, + backend=backend, + cache=cache, + model_id=model_id, + ) + ), + embedding_cache_path=settings.tmp_folder / embedding_cache_name, + ), + ) + if current_training_state().operation.mode is TrainingMode.PREPARE_ONLY: + publish_prepared_features( + settings=settings, + backend_id=ACCURATE_BACKEND_ID, + model_id=prepared.model_id, + device=prepared.runtime_device, + dtype=prepared.runtime_dtype, + utterances=utterances, + x_train=np.asarray(prepared.x_train, dtype=np.float64), + x_test=np.asarray(prepared.x_test, dtype=np.float64), + y_train=prepared.y_train, + y_test=prepared.y_test, + metadata={ + "train_sample_ids": [item.sample_id for item in prepared.train_utterances], + "test_sample_ids": [item.sample_id for item in prepared.test_utterances], + "split_metadata": prepared.split_metadata, + "test_meta": prepared.test_meta, + "train_meta": prepared.train_meta, + }, + cache_namespace=embedding_cache_name, + windowing_policy={ + "pool_window_size_seconds": settings.accurate_runtime.pool_window_size_seconds, + "pool_window_stride_seconds": settings.accurate_runtime.pool_window_stride_seconds, + "pooling_strategy": "mean_std", + }, + noise_statistics={}, + ) + return + if ( + plan is None + and current_training_state().readiness is not None + and current_training_state().utterances + ): + filtered_x, filtered_y, filtered_meta, train_utterances = canonical_train_partition( + settings=settings, + x_train=np.asarray(prepared.x_train, dtype=np.float64), + y_train=prepared.y_train, + train_metadata=prepared.train_meta, + sample_id=lambda item: training_meta_sample_ids([item])[0], + ) + prepared = replace( + prepared, + train_utterances=train_utterances, + x_train=filtered_x, + y_train=filtered_y, + train_meta=filtered_meta, + ) run_prepared_training(prepared, utterances, settings) @@ -134,32 +250,132 @@ def train_accurate_research_profile_model( allowed_restricted_backends=allowed_restricted_backends, persisted_consents=persisted_consents, ) - utterances, prepared = prepare_accurate_research_training( + resolved_model_id = resolve_model_id_for_settings(settings) + resolved_device, resolved_dtype = resolve_runtime_selectors_for_settings(settings) + plan = validate_operation_plan( settings=settings, - logger=logger, - load_utterances_for_training=load_utterances_for_training, - ensure_dataset_consents_for_training=lambda training_utterances: ( - ensure_dataset_consents_for_training(utterances=training_utterances) - ), - split_utterances=split_utterances, - resolve_model_id=lambda: resolve_model_id_for_settings(settings), - resolve_runtime_selectors=lambda: resolve_runtime_selectors_for_settings(settings), - build_backend=lambda model_id, runtime_device, runtime_dtype: build_backend( - model_id, - runtime_device, - runtime_dtype, - settings, - ), - build_feature_dataset=lambda partition, backend, cache, model_id: ( - build_feature_dataset( - utterances=partition, - backend=backend, - cache=cache, - model_id=model_id, - ) - ), - embedding_cache_path=settings.tmp_folder / embedding_cache_name, + backend_id=ACCURATE_RESEARCH_BACKEND_ID, + model_id=resolved_model_id, + device=resolved_device, + dtype=resolved_dtype, ) + if plan is not None: + payload = read_prepared_feature_payload(plan) + metadata = payload.metadata + by_id = {item.sample_id: item for item in current_training_state().utterances} + train_ids = cast(list[str], metadata.get("train_sample_ids", [])) + test_ids = cast(list[str], metadata.get("test_sample_ids", [])) + split_raw = cast(dict[str, object], metadata.get("split_metadata", {})) + test_meta_raw = cast(list[dict[str, str]], metadata.get("test_meta", [])) + train_meta_raw = cast(list[dict[str, str]], metadata.get("train_meta", [])) + utterances = list(current_training_state().utterances) + prepared = cast( + AccurateTrainingPreparation[Utterance, MediumSplitMetadata, _MetaT], + AccurateTrainingPreparation( + train_utterances=[by_id[item] for item in train_ids], + test_utterances=[by_id[item] for item in test_ids], + split_metadata=medium_split_metadata_from_mapping(split_raw), + model_id=resolved_model_id, + runtime_device=resolved_device, + runtime_dtype=resolved_dtype, + x_train=payload.x_train, + y_train=payload.y_train, + train_meta=[WindowMeta(**item) for item in train_meta_raw], + x_test=payload.x_test, + y_test=payload.y_test, + test_meta=[WindowMeta(**item) for item in test_meta_raw], + ), + ) + else: + utterances, prepared = prepare_until_quarantine_stable( + settings=settings, + prepare=lambda: prepare_accurate_research_training( + settings=settings, + logger=logger, + load_utterances_for_training=lambda: list(current_training_state().utterances), + ensure_dataset_consents_for_training=lambda training_utterances: ( + ensure_dataset_consents_for_training(utterances=training_utterances) + ), + split_utterances=split_utterances, + resolve_model_id=lambda: resolved_model_id, + resolve_runtime_selectors=lambda: (resolved_device, resolved_dtype), + build_backend=lambda model_id, runtime_device, runtime_dtype: cast( + Emotion2VecBackend, + reuse_checked_backend( + backend_id=ACCURATE_RESEARCH_BACKEND_ID, + model_id=model_id, + device=runtime_device, + dtype=runtime_dtype, + build=lambda: build_backend( + model_id, + runtime_device, + runtime_dtype, + settings, + ), + ), + ), + build_feature_dataset=lambda partition, backend, cache, model_id: ( + build_feature_dataset( + utterances=partition, + backend=backend, + cache=cache, + model_id=model_id, + ) + ), + embedding_cache_path=settings.tmp_folder / embedding_cache_name, + ), + ) + if current_training_state().operation.mode is TrainingMode.PREPARE_ONLY: + publish_prepared_features( + settings=settings, + backend_id=ACCURATE_RESEARCH_BACKEND_ID, + model_id=prepared.model_id, + device=prepared.runtime_device, + dtype=prepared.runtime_dtype, + utterances=utterances, + x_train=np.asarray(prepared.x_train, dtype=np.float64), + x_test=np.asarray(prepared.x_test, dtype=np.float64), + y_train=prepared.y_train, + y_test=prepared.y_test, + metadata={ + "train_sample_ids": [item.sample_id for item in prepared.train_utterances], + "test_sample_ids": [item.sample_id for item in prepared.test_utterances], + "split_metadata": prepared.split_metadata, + "test_meta": prepared.test_meta, + "train_meta": prepared.train_meta, + }, + cache_namespace=embedding_cache_name, + windowing_policy={ + "pool_window_size_seconds": ( + settings.accurate_research_runtime.pool_window_size_seconds + ), + "pool_window_stride_seconds": ( + settings.accurate_research_runtime.pool_window_stride_seconds + ), + "pooling_strategy": "mean_std", + }, + noise_statistics={}, + ) + return + if ( + plan is None + and current_training_state().readiness is not None + and current_training_state().utterances + ): + filtered_x, filtered_y, filtered_meta, train_utterances = canonical_train_partition( + settings=settings, + x_train=np.asarray(prepared.x_train, dtype=np.float64), + y_train=prepared.y_train, + train_metadata=prepared.train_meta, + sample_id=lambda item: training_meta_sample_ids([item])[0], + ) + prepared = replace( + prepared, + train_utterances=train_utterances, + x_train=filtered_x, + y_train=filtered_y, + train_meta=filtered_meta, + ) run_prepared_training(prepared, utterances, settings) @@ -190,16 +406,14 @@ def prepare_accurate_whisper_training( logger=logger, load_utterances_for_training=load_utterances_for_training, ensure_dataset_consents_for_training=ensure_dataset_consents_for_training, - prepare_training_features=lambda training_utterances: ( - prepare_accurate_training_features( - utterances=training_utterances, - split_utterances=split_utterances, - resolve_model_id=resolve_model_id, - resolve_runtime_selectors=resolve_runtime_selectors, - build_backend=build_backend, - build_cache=lambda: EmbeddingCache(embedding_cache_path), - build_feature_dataset=build_feature_dataset, - ) + prepare_training_features=lambda training_utterances: prepare_accurate_training_features( + utterances=training_utterances, + split_utterances=split_utterances, + resolve_model_id=resolve_model_id, + resolve_runtime_selectors=resolve_runtime_selectors, + build_backend=build_backend, + build_cache=lambda: EmbeddingCache(embedding_cache_path), + build_feature_dataset=build_feature_dataset, ), ) @@ -231,16 +445,14 @@ def prepare_accurate_research_training( logger=logger, load_utterances_for_training=load_utterances_for_training, ensure_dataset_consents_for_training=ensure_dataset_consents_for_training, - prepare_training_features=lambda training_utterances: ( - prepare_accurate_training_features( - utterances=training_utterances, - split_utterances=split_utterances, - resolve_model_id=resolve_model_id, - resolve_runtime_selectors=resolve_runtime_selectors, - build_backend=build_backend, - build_cache=lambda: EmbeddingCache(embedding_cache_path), - build_feature_dataset=build_feature_dataset, - ) + prepare_training_features=lambda training_utterances: prepare_accurate_training_features( + utterances=training_utterances, + split_utterances=split_utterances, + resolve_model_id=resolve_model_id, + resolve_runtime_selectors=resolve_runtime_selectors, + build_backend=build_backend, + build_cache=lambda: EmbeddingCache(embedding_cache_path), + build_feature_dataset=build_feature_dataset, ), ) diff --git a/ser/_internal/models/artifact_persistence.py b/ser/_internal/models/artifact_persistence.py index e28d730..a4f2c19 100644 --- a/ser/_internal/models/artifact_persistence.py +++ b/ser/_internal/models/artifact_persistence.py @@ -2,6 +2,7 @@ from __future__ import annotations +import errno import importlib import json import os @@ -13,12 +14,16 @@ from sklearn.neural_network import MLPClassifier from sklearn.pipeline import Pipeline -from ser._internal.utils.logger import get_logger +from ..utils.logger import get_logger # noqa: TID251 +from .training_readiness import OptionalArtifactError # noqa: TID251 type EmotionClassifier = MLPClassifier | Pipeline _PersistedArtifactsT = TypeVar("_PersistedArtifactsT") logger = get_logger(__name__) +_OPTIONAL_TARGET_ERRNOS = frozenset( + {errno.EACCES, errno.EDQUOT, errno.ENOSPC, errno.EPERM, errno.EROFS} +) class _ModelsConfigLike(Protocol): @@ -38,6 +43,28 @@ class _SettingsLike(Protocol): def models(self) -> _ModelsConfigLike: ... +def _is_optional_target_write_error( + error: OSError, + *, + path: Path, + tmp_path: Path, +) -> bool: + """Returns whether an OSError is a known failure of only the optional target.""" + if error.errno not in _OPTIONAL_TARGET_ERRNOS: + return False + target_paths = { + path.expanduser().resolve(strict=False), + path.parent.expanduser().resolve(strict=False), + tmp_path.expanduser().resolve(strict=False), + } + reported_paths = { + Path(raw).expanduser().resolve(strict=False) + for raw in (error.filename, error.filename2) + if isinstance(raw, str) + } + return bool(reported_paths) and reported_paths.issubset(target_paths) + + def atomic_write_bytes(path: Path, payload: bytes) -> None: """Writes binary payload atomically to avoid partial artifacts.""" path.parent.mkdir(parents=True, exist_ok=True) @@ -68,26 +95,46 @@ def persist_secure_artifact(path: Path, model: EmotionClassifier) -> bool: """Attempts to persist a secure model format via `skops`, if available.""" try: skops_io: Any = importlib.import_module("skops.io") - except ModuleNotFoundError: - logger.info( - "Optional dependency `skops` is not installed; skipping secure model " - "artifact generation." + except ModuleNotFoundError as err: + if err.name not in {"skops", "skops.io"}: + raise + from ser._internal.models.training_orchestration import ( # noqa: TID251 + record_optional_artifact_failure, + ) + + record_optional_artifact_failure( + OptionalArtifactError( + "Optional dependency `skops` is not installed; secure artifact skipped." + ) ) return False - path.parent.mkdir(parents=True, exist_ok=True) tmp_path = path.with_suffix(f"{path.suffix}.tmp") try: + path.parent.mkdir(parents=True, exist_ok=True) skops_io.dump(model, str(tmp_path)) os.replace(tmp_path, path) return True - except Exception as err: - logger.warning( - "Failed to persist secure model artifact at %s: %s. Continuing with " - "pickle artifact.", - path, - err, + except OptionalArtifactError as err: + from ser._internal.models.training_orchestration import ( # noqa: TID251 + record_optional_artifact_failure, + ) + + record_optional_artifact_failure(err) + return False + except OSError as err: + if not _is_optional_target_write_error(err, path=path, tmp_path=tmp_path): + raise + from ser._internal.models.training_orchestration import ( # noqa: TID251 + record_optional_artifact_failure, + ) + + wrapped = OptionalArtifactError( + err.errno, + f"Secure artifact target write failed: {err.strerror or err}", + err.filename, ) + record_optional_artifact_failure(wrapped) return False finally: if tmp_path.exists(): diff --git a/ser/_internal/models/fast_training.py b/ser/_internal/models/fast_training.py index d36b719..02fb69e 100644 --- a/ser/_internal/models/fast_training.py +++ b/ser/_internal/models/fast_training.py @@ -6,13 +6,23 @@ from collections.abc import Callable, Sequence from dataclasses import dataclass from pathlib import Path -from typing import Protocol +from typing import Protocol, cast import numpy as np from sklearn.neural_network import MLPClassifier from sklearn.pipeline import Pipeline from ser._internal.data import Utterance +from ser._internal.models.training_orchestration import ( + build_training_robustness_provenance, + current_training_state, + prepare_until_quarantine_stable, + publish_prepared_features, + read_prepared_feature_payload, + validate_operation_plan, +) +from ser._internal.models.training_readiness import TrainingMode +from ser.config import AppConfig type EmotionClassifier = MLPClassifier | Pipeline type DatasetSplit = tuple[np.ndarray, np.ndarray, list[str], list[str]] @@ -127,6 +137,7 @@ class FastTrainingHooks: persist_training_report: PersistTrainingReportFn default_backend_id: str default_profile_id: str + load_checked_data: Callable[[Sequence[Utterance]], DatasetSplit | None] | None = None def train_fast_model(*, hooks: FastTrainingHooks) -> None: @@ -138,11 +149,67 @@ def train_fast_model(*, hooks: FastTrainingHooks) -> None: utterances=list(utterances_for_consent), ) - data = hooks.load_data(test_size=float(settings.training.test_size)) - if data is None: - hooks.logger.error("Dataset not loaded. Please load the dataset first.") - raise RuntimeError("Dataset not loaded. Please load the dataset first.") - x_train, x_test, y_train, y_test = data + state = current_training_state() + plan = None + if state.operation.prepared_plan is not None: + plan = validate_operation_plan( + settings=cast(AppConfig, settings), + backend_id=hooks.default_backend_id, + model_id="builtin", + device="cpu", + dtype="float64", + ) + if plan is not None: + payload = read_prepared_feature_payload(plan) + x_train, x_test = payload.x_train, payload.x_test + y_train, y_test = payload.y_train, payload.y_test + else: + if hooks.load_checked_data is not None and state.utterances: + checked_loader = hooks.load_checked_data + data = prepare_until_quarantine_stable( + settings=cast(AppConfig, settings), + prepare=lambda: checked_loader(current_training_state().utterances), + ) + else: + data = hooks.load_data(test_size=float(settings.training.test_size)) + if data is None: + hooks.logger.error("Dataset not loaded. Please load the dataset first.") + raise RuntimeError("Dataset not loaded. Please load the dataset first.") + x_train, x_test, y_train, y_test = data + if state.operation.mode is TrainingMode.PREPARE_ONLY: + readiness_utterances = list(state.utterances) + publish_prepared_features( + settings=cast(AppConfig, settings), + backend_id=hooks.default_backend_id, + model_id="builtin", + device="cpu", + dtype="float64", + utterances=readiness_utterances, + x_train=np.asarray(x_train, dtype=np.float64), + x_test=np.asarray(x_test, dtype=np.float64), + y_train=y_train, + y_test=y_test, + metadata={"split_metadata": {"strategy": "fast_data_loader"}}, + cache_namespace="fast_features", + windowing_policy={"strategy": "utterance_handcrafted"}, + noise_statistics={}, + ) + return + if plan is None and state.readiness is not None and state.utterances: + from ser._internal.models.dataset_splitting import split_utterances # noqa: TID251 + from ser._internal.models.training_orchestration import canonical_train_rows # noqa: TID251 + + initial_train, _, _ = split_utterances( + samples=list(state.utterances), + settings=cast(AppConfig, settings), + logger=hooks.logger, + ) + x_train, y_train = canonical_train_rows( + settings=cast(AppConfig, settings), + x_train=np.asarray(x_train, dtype=np.float64), + y_train=y_train, + train_sample_ids=[item.sample_id for item in initial_train], + ) model = hooks.create_classifier() hooks.logger.info(msg="Dataset loaded successfully.") @@ -160,6 +227,10 @@ def train_fast_model(*, hooks: FastTrainingHooks) -> None: backend_id=hooks.default_backend_id, profile=hooks.default_profile_id, ) + provenance = { + **provenance, + "training_robustness": build_training_robustness_provenance(), + } artifact = hooks.build_model_artifact( model=model, feature_vector_size=int(x_train.shape[1]), @@ -173,6 +244,10 @@ def train_fast_model(*, hooks: FastTrainingHooks) -> None: if persisted_artifacts.secure_path is not None: hooks.logger.info(msg=f"Secure model saved to {persisted_artifacts.secure_path}") + report_provenance = { + **provenance, + "training_robustness": build_training_robustness_provenance(), + } report = hooks.build_training_report( accuracy=evaluation.accuracy, macro_f1=evaluation.macro_f1, @@ -183,7 +258,7 @@ def train_fast_model(*, hooks: FastTrainingHooks) -> None: labels=[*y_train, *y_test], artifacts=persisted_artifacts, artifact_metadata=artifact_metadata, - provenance=provenance, + provenance=report_provenance, ) hooks.persist_training_report(report, settings.models.training_report_file) hooks.logger.info(msg=f"Training report saved to {settings.models.training_report_file}") diff --git a/ser/_internal/models/feature_runtime_encoding.py b/ser/_internal/models/feature_runtime_encoding.py index f57fbe8..f494302 100644 --- a/ser/_internal/models/feature_runtime_encoding.py +++ b/ser/_internal/models/feature_runtime_encoding.py @@ -126,12 +126,19 @@ def encode_sequence_with_cache( read_audio: ReadAudioFile = read_audio_file, ) -> EncodedSequence: """Encodes one audio sequence and caches embeddings with runtime selectors.""" + from ser._internal.models.training_orchestration import ( # noqa: TID251 + bounded_retry_local_io, + record_cache_activity, + ) def _compute_sequence() -> EncodedSequence: - audio, sample_rate = read_audio( - file_path=audio_path, - start_seconds=start_seconds, - duration_seconds=duration_seconds, + audio, sample_rate = bounded_retry_local_io( + lambda: read_audio( + file_path=audio_path, + start_seconds=start_seconds, + duration_seconds=duration_seconds, + ), + identity=audio_path, ) audio_array = np.asarray(audio, dtype=np.float32) return backend.encode_sequence(audio_array, sample_rate) @@ -146,6 +153,10 @@ def _compute_sequence() -> EncodedSequence: duration_seconds=duration_seconds, compute=_compute_sequence, ) + record_cache_activity( + cache_hit=cache_entry.cache_hit, + recomputed=cache_entry.invalidated_stale_entry, + ) logger.debug( "%s embedding cache %s for %s (%s).", log_prefix, diff --git a/ser/_internal/models/medium_feature_dataset.py b/ser/_internal/models/medium_feature_dataset.py index 355373b..65156cb 100644 --- a/ser/_internal/models/medium_feature_dataset.py +++ b/ser/_internal/models/medium_feature_dataset.py @@ -5,11 +5,11 @@ import logging from collections.abc import Callable, Sequence from functools import partial -from typing import TypeVar +from typing import TypeVar, cast import numpy as np -from ser._internal.data import EmbeddingCache +from ser._internal.data import EmbeddingCache, Utterance from ser._internal.models.feature_runtime_encoding import ( encode_sequence_with_cache, resolve_effective_model_id, @@ -22,6 +22,7 @@ from ser._internal.models.profile_training_preparation import ( build_medium_feature_dataset as _build_prepared_medium_feature_dataset, ) +from ser._internal.models.training_orchestration import handle_sample_encoding_failure from ser._internal.pool import mean_std_pool, temporal_pooling_windows from ser._internal.repr import EncodedSequence, PoolingWindow, XLSRBackend from ser._internal.utils.audio_utils import read_audio_file @@ -130,6 +131,11 @@ def _encode_utterance(utterance: _UtteranceT) -> EncodedSequence: merge_noise_stats=merge_noise_stats, initial_noise_stats=MediumNoiseControlStats(), window_meta_factory=window_meta_factory, + handle_sample_failure=lambda utterance, error: handle_sample_encoding_failure( + settings=settings, + sample=cast(Utterance, utterance), + error=error, + ), ) diff --git a/ser/_internal/models/medium_training_preparation.py b/ser/_internal/models/medium_training_preparation.py index c4639a5..576a42e 100644 --- a/ser/_internal/models/medium_training_preparation.py +++ b/ser/_internal/models/medium_training_preparation.py @@ -4,18 +4,39 @@ import logging from collections.abc import Callable +from dataclasses import replace from pathlib import Path -from typing import Protocol, TypeVar +from typing import Protocol, TypeVar, cast import numpy as np from ser._internal.data import EmbeddingCache, Utterance -from ser._internal.models.dataset_splitting import MediumSplitMetadata +from ser._internal.models.dataset_splitting import ( + MediumSplitMetadata, + medium_split_metadata_from_mapping, +) +from ser._internal.models.medium_noise_controls import ( + MediumNoiseControlStats, + merge_medium_noise_stats, +) from ser._internal.models.training_execution import run_medium_profile_training +from ser._internal.models.training_orchestration import ( + canonical_train_partition, + current_training_state, + prepare_until_quarantine_stable, + publish_prepared_features, + read_prepared_feature_payload, + record_dropped_windows, + reuse_checked_backend, + training_meta_sample_ids, + validate_operation_plan, +) from ser._internal.models.training_preparation import ( prepare_medium_training_features, prepare_medium_training_payload, ) +from ser._internal.models.training_readiness import TrainingMode +from ser._internal.models.training_support import WindowMeta from ser._internal.models.training_types import ( MediumTrainingPreparation, PersistedArtifactsLike, @@ -38,6 +59,23 @@ def __call__(self, *, settings: AppConfig, backend_id: str) -> tuple[str, str]: type TrainMediumProfileModelCallable = Callable[..., None] +def _partition_noise_stats(sample_ids: set[str]) -> MediumNoiseControlStats: + """Aggregates exact per-sample medium noise stats for one canonical partition.""" + aggregate = MediumNoiseControlStats() + by_sample = current_training_state().medium_noise_stats_by_sample + missing = sample_ids.difference(by_sample) + if missing: + raise RuntimeError( + f"Medium noise provenance is missing {len(missing)} canonical sample(s)." + ) + for sample_id in sorted(sample_ids): + stats = by_sample[sample_id] + if not isinstance(stats, MediumNoiseControlStats): + raise TypeError("Medium noise provenance contains an invalid statistics payload.") + aggregate = merge_medium_noise_stats(aggregate, stats) + return aggregate + + def prepare_medium_xlsr_training( *, settings: AppConfig, @@ -70,16 +108,14 @@ def prepare_medium_xlsr_training( logger=logger, load_utterances_for_training=load_utterances_for_training, ensure_dataset_consents_for_training=ensure_dataset_consents_for_training, - prepare_training_features=lambda training_utterances: ( - prepare_medium_training_features( - utterances=training_utterances, - split_utterances=split_utterances, - resolve_model_id=resolve_model_id, - resolve_runtime_selectors=resolve_runtime_selectors, - build_backend=build_backend, - build_cache=lambda: EmbeddingCache(embedding_cache_path), - build_feature_dataset=build_feature_dataset, - ) + prepare_training_features=lambda training_utterances: prepare_medium_training_features( + utterances=training_utterances, + split_utterances=split_utterances, + resolve_model_id=resolve_model_id, + resolve_runtime_selectors=resolve_runtime_selectors, + build_backend=build_backend, + build_cache=lambda: EmbeddingCache(embedding_cache_path), + build_feature_dataset=build_feature_dataset, ), ) @@ -188,34 +224,166 @@ def train_medium_profile_model( embedding_cache_name: str = "medium_embeddings", ) -> None: """Runs medium-profile training from settings using delegated hooks.""" - utterances, prepared = prepare_medium_xlsr_training( + resolved_model_id = resolve_model_id_for_settings(settings) + resolved_device, resolved_dtype = resolve_runtime_selectors_for_settings(settings) + plan = validate_operation_plan( settings=settings, - logger=logger, - load_utterances_for_training=load_utterances_for_training, - ensure_dataset_consents_for_training=lambda training_utterances: ( - ensure_dataset_consents_for_training(utterances=training_utterances) - ), - split_utterances=split_utterances, - resolve_model_id=lambda: resolve_model_id_for_settings(settings), - resolve_runtime_selectors=lambda: resolve_runtime_selectors_for_settings(settings), - build_backend=lambda model_id, runtime_device, runtime_dtype: build_backend( - model_id, - runtime_device, - runtime_dtype, - settings, - ), - build_feature_dataset=lambda partition, backend, cache, model_id: ( - build_feature_dataset( - utterances=partition, - backend=backend, - cache=cache, - model_id=model_id, - ) - ), - embedding_cache_path=settings.tmp_folder / embedding_cache_name, + backend_id=backend_id, + model_id=resolved_model_id, + device=resolved_device, + dtype=resolved_dtype, + ) + if plan is not None: + payload = read_prepared_feature_payload(plan) + metadata = payload.metadata + by_id = {item.sample_id: item for item in current_training_state().utterances} + train_ids = cast(list[str], metadata.get("train_sample_ids", [])) + test_ids = cast(list[str], metadata.get("test_sample_ids", [])) + split_raw = cast(dict[str, object], metadata.get("split_metadata", {})) + train_stats_raw = cast(dict[str, int], metadata.get("train_noise_stats", {})) + test_stats_raw = cast(dict[str, int], metadata.get("test_noise_stats", {})) + test_meta_raw = cast(list[dict[str, str]], metadata.get("test_meta", [])) + train_meta_raw = cast(list[dict[str, str]], metadata.get("train_meta", [])) + utterances = list(current_training_state().utterances) + prepared = cast( + MediumTrainingPreparation[Utterance, MediumSplitMetadata, _MetaT, _NoiseStatsT], + MediumTrainingPreparation( + train_utterances=[by_id[item] for item in train_ids], + test_utterances=[by_id[item] for item in test_ids], + split_metadata=medium_split_metadata_from_mapping(split_raw), + model_id=resolved_model_id, + runtime_device=resolved_device, + runtime_dtype=resolved_dtype, + x_train=payload.x_train, + y_train=payload.y_train, + train_meta=[WindowMeta(**item) for item in train_meta_raw], + x_test=payload.x_test, + y_test=payload.y_test, + test_meta=[WindowMeta(**item) for item in test_meta_raw], + train_noise_stats=MediumNoiseControlStats(**train_stats_raw), + test_noise_stats=MediumNoiseControlStats(**test_stats_raw), + ), + ) + else: + utterances, prepared = prepare_until_quarantine_stable( + settings=settings, + prepare=lambda: prepare_medium_xlsr_training( + settings=settings, + logger=logger, + load_utterances_for_training=lambda: list(current_training_state().utterances), + ensure_dataset_consents_for_training=lambda training_utterances: ( + ensure_dataset_consents_for_training(utterances=training_utterances) + ), + split_utterances=split_utterances, + resolve_model_id=lambda: resolved_model_id, + resolve_runtime_selectors=lambda: (resolved_device, resolved_dtype), + build_backend=lambda model_id, runtime_device, runtime_dtype: cast( + XLSRBackend, + reuse_checked_backend( + backend_id=backend_id, + model_id=model_id, + device=runtime_device, + dtype=runtime_dtype, + build=lambda: build_backend( + model_id, + runtime_device, + runtime_dtype, + settings, + ), + ), + ), + build_feature_dataset=lambda partition, backend, cache, model_id: ( + build_feature_dataset( + utterances=partition, + backend=backend, + cache=cache, + model_id=model_id, + ) + ), + embedding_cache_path=settings.tmp_folder / embedding_cache_name, + ), + ) + canonical_prepared = prepared + dev_noise_stats: MediumNoiseControlStats | None = None + if ( + plan is None + and current_training_state().readiness is not None + and current_training_state().utterances + ): + filtered_x, filtered_y, filtered_meta, train_utterances = canonical_train_partition( + settings=settings, + x_train=np.asarray(prepared.x_train, dtype=np.float64), + y_train=prepared.y_train, + train_metadata=prepared.train_meta, + sample_id=lambda item: training_meta_sample_ids([item])[0], + ) + canonical_train_ids = {item.sample_id for item in train_utterances} + dev_ids = { + item.sample_id + for item in prepared.train_utterances + if item.sample_id not in canonical_train_ids + } + canonical_prepared = replace( + prepared, + train_utterances=train_utterances, + x_train=filtered_x, + y_train=filtered_y, + train_meta=filtered_meta, + train_noise_stats=cast( + _NoiseStatsT, + _partition_noise_stats(canonical_train_ids), + ), + ) + dev_noise_stats = _partition_noise_stats(dev_ids) + dropped_windows = 0 + run_noise_stats: tuple[object, ...] = ( + getattr(canonical_prepared, "train_noise_stats", None), + getattr(prepared, "test_noise_stats", None), ) + if current_training_state().operation.mode is TrainingMode.PREPARE_ONLY: + run_noise_stats = (*run_noise_stats, dev_noise_stats) + for noise_stats in run_noise_stats: + if isinstance(noise_stats, MediumNoiseControlStats): + dropped_windows += noise_stats.dropped_low_std_windows + dropped_windows += noise_stats.dropped_cap_windows + record_dropped_windows(dropped_windows) + if current_training_state().operation.mode is TrainingMode.PREPARE_ONLY: + publish_prepared_features( + settings=settings, + backend_id=backend_id, + model_id=prepared.model_id, + device=prepared.runtime_device, + dtype=prepared.runtime_dtype, + utterances=utterances, + x_train=np.asarray(prepared.x_train, dtype=np.float64), + x_test=np.asarray(prepared.x_test, dtype=np.float64), + y_train=prepared.y_train, + y_test=prepared.y_test, + metadata={ + "train_sample_ids": [item.sample_id for item in prepared.train_utterances], + "test_sample_ids": [item.sample_id for item in prepared.test_utterances], + "split_metadata": prepared.split_metadata, + "test_meta": prepared.test_meta, + "train_meta": prepared.train_meta, + "train_noise_stats": canonical_prepared.train_noise_stats, + "dev_noise_stats": dev_noise_stats, + "test_noise_stats": prepared.test_noise_stats, + }, + cache_namespace=embedding_cache_name, + windowing_policy={ + "pool_window_size_seconds": settings.medium_runtime.pool_window_size_seconds, + "pool_window_stride_seconds": settings.medium_runtime.pool_window_stride_seconds, + "pooling_strategy": pooling_strategy, + }, + noise_statistics={ + "train": canonical_prepared.train_noise_stats, + "dev": dev_noise_stats, + "test": prepared.test_noise_stats, + }, + ) + return _ = run_medium_profile_training_from_prepared( - prepared=prepared, + prepared=canonical_prepared, utterances=utterances, settings=settings, logger=logger, diff --git a/ser/_internal/models/profile_training_preparation.py b/ser/_internal/models/profile_training_preparation.py index 18b8930..86ac371 100644 --- a/ser/_internal/models/profile_training_preparation.py +++ b/ser/_internal/models/profile_training_preparation.py @@ -28,7 +28,7 @@ def start_seconds(self) -> float | None: ... def duration_seconds(self) -> float | None: ... @property - def label(self) -> str: ... + def label(self) -> str | None: ... @property def language(self) -> str | None: ... @@ -55,6 +55,7 @@ def build_medium_feature_dataset( merge_noise_stats: Callable[[_StatsT, _StatsT], _StatsT], initial_noise_stats: _StatsT, window_meta_factory: Callable[[str, str, str], _MetaT], + handle_sample_failure: Callable[[_UtteranceT, Exception], bool] | None = None, ) -> tuple[np.ndarray, list[str], list[_MetaT], _StatsT]: """Builds a medium-profile feature matrix from utterance-level embeddings.""" feature_blocks: list[np.ndarray] = [] @@ -62,8 +63,24 @@ def build_medium_feature_dataset( meta: list[_MetaT] = [] aggregate_stats = initial_noise_stats - for utterance in utterances: - encoded = encode_sequence(utterance) + from ser._internal.models.training_orchestration import ( # noqa: TID251 + record_medium_noise_stats, + record_preparation_progress, + ) + + total = len(utterances) + for processed, utterance in enumerate(utterances, start=1): + try: + encoded = encode_sequence(utterance) + except Exception as error: + if handle_sample_failure is not None and handle_sample_failure(utterance, error): + record_preparation_progress( + processed=processed, + total=total, + sample_id=utterance.sample_id, + ) + continue + raise windows = build_pooling_windows( encoded, window_size_seconds, @@ -73,12 +90,20 @@ def build_medium_feature_dataset( filtered, stats = apply_noise_controls(np.asarray(pooled, dtype=np.float64)) feature_blocks.append(filtered) row_count = int(filtered.shape[0]) + if utterance.label is None: + raise ValueError(f"Utterance {utterance.sample_id!r} has no primary emotion target.") labels.extend([utterance.label] * row_count) language = utterance.language or "unknown" meta.extend( [window_meta_factory(utterance.sample_id, utterance.corpus, language)] * row_count ) aggregate_stats = merge_noise_stats(aggregate_stats, stats) + record_medium_noise_stats(sample_id=utterance.sample_id, stats=stats) + record_preparation_progress( + processed=processed, + total=total, + sample_id=utterance.sample_id, + ) if not feature_blocks: raise RuntimeError("Medium training produced no feature vectors.") @@ -100,14 +125,30 @@ def build_accurate_feature_dataset( ], pool_features: Callable[[EncodedSequence, Sequence[PoolingWindow]], np.ndarray], window_meta_factory: Callable[[str, str, str], _MetaT], + handle_sample_failure: Callable[[_UtteranceT, Exception], bool] | None = None, ) -> tuple[np.ndarray, list[str], list[_MetaT]]: """Builds an accurate-profile feature matrix from utterance embeddings.""" feature_blocks: list[np.ndarray] = [] labels: list[str] = [] meta: list[_MetaT] = [] - for utterance in utterances: - encoded = encode_sequence(utterance) + from ser._internal.models.training_orchestration import ( # noqa: TID251 + record_preparation_progress, + ) + + total = len(utterances) + for processed, utterance in enumerate(utterances, start=1): + try: + encoded = encode_sequence(utterance) + except Exception as error: + if handle_sample_failure is not None and handle_sample_failure(utterance, error): + record_preparation_progress( + processed=processed, + total=total, + sample_id=utterance.sample_id, + ) + continue + raise windows = build_pooling_windows( encoded, window_size_seconds, @@ -116,11 +157,18 @@ def build_accurate_feature_dataset( pooled = pool_features(encoded, windows) feature_blocks.append(np.asarray(pooled, dtype=np.float64)) row_count = int(pooled.shape[0]) + if utterance.label is None: + raise ValueError(f"Utterance {utterance.sample_id!r} has no primary emotion target.") labels.extend([utterance.label] * row_count) language = utterance.language or "unknown" meta.extend( [window_meta_factory(utterance.sample_id, utterance.corpus, language)] * row_count ) + record_preparation_progress( + processed=processed, + total=total, + sample_id=utterance.sample_id, + ) if not feature_blocks: raise RuntimeError("Accurate training produced no feature vectors.") diff --git a/ser/_internal/models/training_entrypoints.py b/ser/_internal/models/training_entrypoints.py index 420e78f..7c6ba1f 100644 --- a/ser/_internal/models/training_entrypoints.py +++ b/ser/_internal/models/training_entrypoints.py @@ -18,6 +18,8 @@ import ser._internal.models.medium_noise_controls as _medium_noise_controls import ser._internal.models.medium_training_preparation as _medium_training_preparation import ser._internal.models.profile_runtime as _profile_runtime +import ser._internal.models.training_orchestration as _training_orchestration +import ser._internal.models.training_readiness as _training_readiness import ser._internal.models.training_reporting as _training_reporting import ser._internal.models.training_support as _training_support from ser._internal.runtime.environment_plan import build_runtime_environment_plan @@ -31,17 +33,51 @@ def train_model(*, settings: AppConfig) -> None: """Runs fast-profile training with one explicit settings snapshot.""" + if not _training_orchestration.training_operation_active(): + with _training_orchestration.training_operation_scope( + _training_readiness.TrainingOperation() + ): + train_model(settings=settings) + return + + _training_orchestration.ensure_entrypoint_readiness( + settings=settings, + load_utterances=partial( + _data_loader.load_utterances, + settings=settings, + allow_prepare=False, + ), + ) + if ( + _training_orchestration.current_training_state().operation.mode + is _training_readiness.TrainingMode.DRY_RUN + ): + return + def _build_hooks(active_settings: AppConfig) -> _fast_training.FastTrainingHooks: return _fast_training.FastTrainingHooks( logger=logger, settings=active_settings, - load_utterances=partial(_data_loader.load_utterances, settings=active_settings), + load_utterances=lambda: list( + _training_orchestration.current_training_state().utterances + ), ensure_dataset_consents_for_training=partial( _training_support.ensure_dataset_consents_for_training, settings=active_settings, logger=logger, ), load_data=partial(_data_loader.load_data, settings=active_settings), + load_checked_data=lambda utterances: _data_loader.load_checked_fast_data( + utterances=utterances, + settings=active_settings, + handle_sample_failure=lambda sample, error: ( + _training_orchestration.handle_sample_encoding_failure( + settings=active_settings, + sample=sample, + error=error, + ) + ), + ), create_classifier=lambda: _training_support.create_classifier(active_settings), evaluate_training_predictions=_training_support.evaluate_training_predictions, build_provenance_metadata=_license_check.build_provenance_metadata, @@ -75,11 +111,34 @@ def _build_hooks(active_settings: AppConfig) -> _fast_training.FastTrainingHooks def train_medium_model(*, settings: AppConfig) -> None: """Runs medium-profile training with one explicit settings snapshot.""" + if not _training_orchestration.training_operation_active(): + with _training_orchestration.training_operation_scope( + _training_readiness.TrainingOperation() + ): + train_medium_model(settings=settings) + return + + _training_orchestration.ensure_entrypoint_readiness( + settings=settings, + load_utterances=partial( + _data_loader.load_utterances, + settings=settings, + allow_prepare=False, + ), + ) + if ( + _training_orchestration.current_training_state().operation.mode + is _training_readiness.TrainingMode.DRY_RUN + ): + return + _medium_training_preparation.train_medium_profile_entrypoint( settings=settings, logger=logger, train_profile_model=_medium_training_preparation.train_medium_profile_model, - load_utterances_for_training=partial(_data_loader.load_utterances, settings=settings), + load_utterances_for_training=lambda: list( + _training_orchestration.current_training_state().utterances + ), ensure_dataset_consents_for_training=partial( _training_support.ensure_dataset_consents_for_training, settings=settings, @@ -149,11 +208,34 @@ def train_medium_model(*, settings: AppConfig) -> None: def train_accurate_model(*, settings: AppConfig) -> None: """Runs accurate-profile training with one explicit settings snapshot.""" + if not _training_orchestration.training_operation_active(): + with _training_orchestration.training_operation_scope( + _training_readiness.TrainingOperation() + ): + train_accurate_model(settings=settings) + return + + _training_orchestration.ensure_entrypoint_readiness( + settings=settings, + load_utterances=partial( + _data_loader.load_utterances, + settings=settings, + allow_prepare=False, + ), + ) + if ( + _training_orchestration.current_training_state().operation.mode + is _training_readiness.TrainingMode.DRY_RUN + ): + return + _accurate_training_preparation.train_accurate_whisper_profile_entrypoint( settings=settings, logger=logger, train_profile_model=_accurate_training_preparation.train_accurate_whisper_profile_model, - load_utterances_for_training=partial(_data_loader.load_utterances, settings=settings), + load_utterances_for_training=lambda: list( + _training_orchestration.current_training_state().utterances + ), ensure_dataset_consents_for_training=partial( _training_support.ensure_dataset_consents_for_training, settings=settings, @@ -198,6 +280,35 @@ def train_accurate_model(*, settings: AppConfig) -> None: def train_accurate_research_model(*, settings: AppConfig) -> None: """Runs accurate-research training with one explicit settings snapshot.""" + if not _training_orchestration.training_operation_active(): + with _training_orchestration.training_operation_scope( + _training_readiness.TrainingOperation() + ): + train_accurate_research_model(settings=settings) + return + + allowed_restricted_backends = _license_check.parse_allowed_restricted_backends_env() + persisted_consents = _license_check.load_persisted_backend_consents(settings=settings) + _license_check.ensure_backend_access( + backend_id=_profile_runtime.ACCURATE_RESEARCH_BACKEND_ID, + restricted_backends_enabled=settings.runtime_flags.restricted_backends, + allowed_restricted_backends=allowed_restricted_backends, + persisted_consents=persisted_consents, + ) + _training_orchestration.ensure_entrypoint_readiness( + settings=settings, + load_utterances=partial( + _data_loader.load_utterances, + settings=settings, + allow_prepare=False, + ), + ) + if ( + _training_orchestration.current_training_state().operation.mode + is _training_readiness.TrainingMode.DRY_RUN + ): + return + _accurate_training_preparation.train_accurate_research_profile_entrypoint( settings=settings, logger=logger, @@ -208,7 +319,9 @@ def train_accurate_research_model(*, settings: AppConfig) -> None: load_persisted_backend_consents=_license_check.load_persisted_backend_consents, ensure_backend_access=_license_check.ensure_backend_access, restricted_backend_id=_profile_runtime.ACCURATE_RESEARCH_BACKEND_ID, - load_utterances_for_training=partial(_data_loader.load_utterances, settings=settings), + load_utterances_for_training=lambda: list( + _training_orchestration.current_training_state().utterances + ), ensure_dataset_consents_for_training=partial( _training_support.ensure_dataset_consents_for_training, settings=settings, diff --git a/ser/_internal/models/training_execution.py b/ser/_internal/models/training_execution.py index 4564d8e..a389590 100644 --- a/ser/_internal/models/training_execution.py +++ b/ser/_internal/models/training_execution.py @@ -9,6 +9,7 @@ import numpy as np +from ser._internal.models.training_orchestration import build_training_robustness_provenance from ser._internal.models.training_types import ( AccurateTrainingPreparation, MediumTrainingPreparation, @@ -197,6 +198,10 @@ def finalize_profile_training_report( profile_label, persisted_artifacts.secure_path, ) + report_provenance = { + **provenance, + "training_robustness": build_training_robustness_provenance(), + } report = build_training_report( accuracy=evaluation.accuracy, macro_f1=evaluation.macro_f1, @@ -207,7 +212,7 @@ def finalize_profile_training_report( labels=[*y_train, *y_test], artifacts=persisted_artifacts, artifact_metadata=artifact_metadata, - provenance=provenance, + provenance=report_provenance, data_controls=data_controls, ) persist_training_report(report) @@ -262,6 +267,10 @@ def run_medium_profile_training( backend_id=backend_id, profile=profile_id, ) + provenance = { + **provenance, + "training_robustness": build_training_robustness_provenance(), + } execution = execute_default_profile_training( create_classifier=create_classifier, x_train=prepared.x_train, @@ -371,6 +380,10 @@ def run_accurate_profile_training( backend_id=backend_id, profile=profile_id, ) + provenance = { + **provenance, + "training_robustness": build_training_robustness_provenance(), + } execution = execute_default_profile_training( create_classifier=create_classifier, x_train=prepared.x_train, diff --git a/ser/_internal/models/training_orchestration.py b/ser/_internal/models/training_orchestration.py new file mode 100644 index 0000000..26d3e5e --- /dev/null +++ b/ser/_internal/models/training_orchestration.py @@ -0,0 +1,1400 @@ +"""Shared training-operation context, backend smoke, and prepared-feature storage.""" + +from __future__ import annotations + +import json +import os +import signal +import tempfile +import threading +import time +from collections import Counter +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from contextvars import ContextVar, Token +from dataclasses import asdict, dataclass, field, is_dataclass +from datetime import UTC, datetime +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path +from typing import TypeVar, cast + +import numpy as np +from numpy.typing import NDArray + +from ser.config import AppConfig +from ser.profiles import resolve_profile_name + +from ..data import EmbeddingCache, Utterance # noqa: TID251 +from ..features.feature_extractor import _extract_feature_for_settings # noqa: TID251 +from ..pool import mean_std_pool, temporal_pooling_windows # noqa: TID251 +from ..repr import FeatureBackend # noqa: TID251 +from ..utils.audio_utils import read_audio_file # noqa: TID251 +from ..utils.logger import get_logger # noqa: TID251 +from .feature_runtime_encoding import encode_sequence_with_cache # noqa: TID251 +from .profile_runtime import ( # noqa: TID251 + ACCURATE_BACKEND_ID, + ACCURATE_RESEARCH_BACKEND_ID, + MEDIUM_BACKEND_ID, + build_accurate_backend_for_settings, + build_accurate_research_backend_for_settings, + build_medium_backend_for_settings, + resolve_accurate_model_id, + resolve_accurate_research_model_id, + resolve_medium_model_id, + resolve_runtime_selectors_for_backend_id, +) +from .training_readiness import ( # noqa: TID251 + PREPARATION_CODE_VERSION, + PREPARED_CACHE_VERSION, + FailureDisposition, + FailureScope, + PreparedPlan, + PreparedPlanError, + QuarantineBudgetExceeded, + QuarantinePolicy, + QuarantineRecord, + ReadinessReport, + SmokeResult, + TrainingMode, + TrainingOperation, + TrainingReadinessError, + _allowed_media_roots, + build_prepared_plan, + build_quarantine_record, + classify_failure, + default_prepared_payload_path, + default_prepared_plan_path, + digest_payload, + enforce_quarantine_budget, + expected_cache_namespace, + hash_file, + load_prepared_plan, + quarantine_ledger_digest, + run_training_readiness, + validate_prepared_plan, + validated_cache_root, + write_prepared_plan, + write_quarantine_ledger, +) + +logger = get_logger(__name__) +_PreparationT = TypeVar("_PreparationT") + + +@dataclass(slots=True) +class TrainingRunState: + """Mutable process-local state shared across one training orchestration.""" + + operation: TrainingOperation + readiness: ReadinessReport | None = None + utterances: tuple[Utterance, ...] = () + quarantine_records: list[QuarantineRecord] = field(default_factory=list) + checked_backend: FeatureBackend | None = None + checked_backend_id: str | None = None + checked_model_id: str | None = None + checked_device: str | None = None + checked_dtype: str | None = None + cache_hits: int = 0 + cache_misses: int = 0 + recomputed_cache_entries: int = 0 + dropped_windows: int = 0 + bounded_retries: int = 0 + preparation_quarantine_changed: bool = False + preparation_started_at: float | None = None + containment_counts: Counter[str] = field(default_factory=Counter) + medium_noise_stats_by_sample: dict[str, object] = field(default_factory=dict) + + +@dataclass(frozen=True, slots=True) +class PreparedFeaturePayload: + """Safe non-pickle representation of reusable feature preparation.""" + + x_train: NDArray[np.float64] + x_dev: NDArray[np.float64] + x_test: NDArray[np.float64] + y_train: list[str] + y_dev: list[str] + y_test: list[str] + metadata: Mapping[str, object] + + +_DEFAULT_STATE = TrainingRunState(operation=TrainingOperation()) +_TRAINING_STATE: ContextVar[TrainingRunState] = ContextVar( + "ser_training_run_state", + default=_DEFAULT_STATE, +) + + +def current_training_state() -> TrainingRunState: + """Returns the active operation state or the default real-training state.""" + return _TRAINING_STATE.get() + + +def training_operation_active() -> bool: + """Returns whether a caller established an explicit invocation scope.""" + return current_training_state() is not _DEFAULT_STATE + + +def _close_backend(backend: object) -> None: + """Closes one backend when it exposes a deterministic lifecycle hook.""" + for method_name in ("close", "shutdown"): + method = getattr(backend, method_name, None) + if callable(method): + method() + return + + +def close_checked_backend(state: TrainingRunState | None = None) -> None: + """Releases the checked backend and clears all retained runtime references.""" + active_state = state or current_training_state() + backend = active_state.checked_backend + try: + if backend is not None: + _close_backend(backend) + finally: + active_state.checked_backend = None + active_state.checked_backend_id = None + active_state.checked_model_id = None + active_state.checked_device = None + active_state.checked_dtype = None + + +@contextmanager +def training_operation_scope(operation: TrainingOperation) -> Iterator[TrainingRunState]: + """Activates one validated training operation for nested profile entrypoints.""" + operation.validate() + state = TrainingRunState(operation=operation) + token: Token[TrainingRunState] = _TRAINING_STATE.set(state) + try: + yield state + finally: + close_checked_backend(state) + _TRAINING_STATE.reset(token) + + +def _package_version() -> str: + """Returns installed package version with a source-tree fallback.""" + try: + return version("ser") + except PackageNotFoundError: + return "1.0.0" + + +def _recipe_digest(settings: AppConfig) -> str: + """Returns the current recipe content/identifier digest.""" + recipe = settings.dataset.recipe or "none" + recipe_path = Path(recipe).expanduser() + return ( + digest_payload(recipe_path.read_text(encoding="utf-8")) + if recipe_path.is_file() + else digest_payload(recipe) + ) + + +def _cache_keys(settings: AppConfig, namespace: str) -> tuple[str, ...]: + """Returns current cache-relative keys bound to their exact content digests.""" + root = validated_cache_root(settings, namespace) + return tuple( + sorted( + f"{path.relative_to(root)}:{hash_file(path)}" + for path in root.rglob("*.npz") + if path.is_file() + ) + ) + + +def _effective_state_utterances() -> tuple[Utterance, ...]: + """Returns the invocation view after readiness and preparation quarantine.""" + state = current_training_state() + excluded = {record.sample_id for record in state.quarantine_records} + return tuple(item for item in state.utterances if item.sample_id not in excluded) + + +def _current_split_digest(settings: AppConfig) -> str: + """Recomputes the deterministic split ledger for current effective samples.""" + from ser._internal.models.dataset_splitting import ( # noqa: TID251 + split_utterances_three_way, + ) + + train, dev, test, metadata = split_utterances_three_way( + samples=list(_effective_state_utterances()), + settings=settings, + logger=logger, + ) + return digest_payload( + { + "metadata": asdict(metadata), + "train_sample_ids": sorted(item.sample_id for item in train), + "dev_sample_ids": sorted(item.sample_id for item in dev), + "test_sample_ids": sorted(item.sample_id for item in test), + } + ) + + +def _current_quarantine_digest() -> str: + """Returns the digest of the complete deterministic quarantine ledger.""" + return quarantine_ledger_digest(current_training_state().quarantine_records) + + +def _resolved_model_revision(*, backend_id: str, model_id: str) -> str: + """Returns an exact backend-resolved revision or rejects unverifiable reuse.""" + if backend_id == "handcrafted" and model_id == "builtin": + return PREPARATION_CODE_VERSION + state = current_training_state() + backend = state.checked_backend + if ( + backend is None + or state.checked_backend_id != backend_id + or state.checked_model_id != model_id + ): + raise PreparedPlanError( + "Prepared plan requires the exact readiness-checked backend instance." + ) + revision = getattr(backend, "model_revision", None) + if not isinstance(revision, str) or not revision.strip() or revision.strip() == model_id: + raise PreparedPlanError( + "Prepared plan backend revision is unpinned or could not be verified." + ) + return revision.strip() + + +def _profile_smoke_runtime( + settings: AppConfig, +) -> tuple[str, str, str, str, FeatureBackend]: + """Resolves and builds the exact selected feature backend for smoke validation.""" + profile = resolve_profile_name(settings) + if profile == "medium": + backend_id = MEDIUM_BACKEND_ID + model_id = resolve_medium_model_id(settings) + device, dtype = resolve_runtime_selectors_for_backend_id( + settings=settings, + backend_id=backend_id, + logger=logger, + ) + return ( + backend_id, + model_id, + device, + dtype, + build_medium_backend_for_settings(model_id, device, dtype, settings), + ) + if profile == "accurate": + backend_id = ACCURATE_BACKEND_ID + model_id = resolve_accurate_model_id(settings) + device, dtype = resolve_runtime_selectors_for_backend_id( + settings=settings, + backend_id=backend_id, + logger=logger, + ) + return ( + backend_id, + model_id, + device, + dtype, + build_accurate_backend_for_settings(model_id, device, dtype, settings), + ) + if profile == "accurate-research": + backend_id = ACCURATE_RESEARCH_BACKEND_ID + model_id = resolve_accurate_research_model_id(settings) + device, dtype = resolve_runtime_selectors_for_backend_id( + settings=settings, + backend_id=backend_id, + logger=logger, + ) + return ( + backend_id, + model_id, + device, + dtype, + build_accurate_research_backend_for_settings(model_id, device, settings), + ) + raise ValueError(f"Profile {profile!r} does not use a sequence backend.") + + +def _run_selected_backend_smoke_unbounded( + *, + settings: AppConfig, + samples: tuple[Utterance, ...], + probe_cache_dir: Path, +) -> SmokeResult: + """Runs bounded real audio/backend/pooling/cache validation without production writes.""" + if not samples: + raise RuntimeError("Backend smoke requires at least one selected sample.") + profile = resolve_profile_name(settings) + if profile == "fast": + features = [ + np.asarray( + _extract_feature_for_settings( + str(sample.audio_path), + feature_flags=settings.feature_flags, + audio_read_config=settings.audio_read, + ), + dtype=np.float64, + ) + for sample in samples + ] + if any( + vector.ndim != 1 or vector.size <= 0 or not np.all(np.isfinite(vector)) + for vector in features + ): + raise ValueError("Handcrafted smoke feature output is non-finite or shape-invalid.") + probe_path = probe_cache_dir / "handcrafted-smoke.npz" + with probe_path.open("wb") as handle: + np.savez_compressed(handle, features=np.vstack(features)) + with np.load(probe_path, allow_pickle=False) as payload: + round_trip = np.asarray(payload["features"], dtype=np.float64) + expected = np.vstack(features) + return SmokeResult( + attempted=len(samples), + succeeded=len(samples), + feature_dim=int(expected.shape[1]), + cache_round_trip=bool(np.array_equal(round_trip, expected)), + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + ) + + backend_id, model_id, device, dtype, backend_object = _profile_smoke_runtime(settings) + state = current_training_state() + if state.checked_backend is not None: + close_checked_backend(state) + state.checked_backend = backend_object + state.checked_backend_id = backend_id + state.checked_model_id = model_id + state.checked_device = device + state.checked_dtype = dtype + runtime_config = ( + settings.medium_runtime + if profile == "medium" + else ( + settings.accurate_runtime + if profile == "accurate" + else settings.accurate_research_runtime + ) + ) + cache = EmbeddingCache(probe_cache_dir / "embeddings") + pooled_rows: list[NDArray[np.float64]] = [] + cache_round_trip = True + for sample in samples: + + def _read_probe( + file_path: str, + *, + start_seconds: float | None = None, + duration_seconds: float | None = None, + current_sample: Utterance = sample, + ) -> tuple[NDArray[np.float32], int]: + del file_path + return read_audio_file( + str(current_sample.audio_path), + start_seconds=start_seconds, + duration_seconds=duration_seconds, + audio_read_config=settings.audio_read, + ) + + encoded = encode_sequence_with_cache( + audio_path=str(sample.audio_path), + start_seconds=sample.start_seconds, + duration_seconds=sample.duration_seconds, + backend=backend_object, + cache=cache, + backend_id=backend_id, + model_id=model_id, + frame_size_seconds=runtime_config.pool_window_size_seconds, + frame_stride_seconds=runtime_config.pool_window_stride_seconds, + log_prefix="Readiness smoke", + logger=logger, + read_audio=_read_probe, + ) + windows = temporal_pooling_windows( + encoded, + window_size_seconds=runtime_config.pool_window_size_seconds, + window_stride_seconds=runtime_config.pool_window_stride_seconds, + ) + pooled = np.asarray(mean_std_pool(encoded, windows), dtype=np.float64) + if pooled.ndim != 2 or pooled.shape[0] <= 0 or not np.all(np.isfinite(pooled)): + raise ValueError("Backend smoke pooled feature output is non-finite or shape-invalid.") + pooled_rows.append(pooled) + second = encode_sequence_with_cache( + audio_path=str(sample.audio_path), + start_seconds=sample.start_seconds, + duration_seconds=sample.duration_seconds, + backend=backend_object, + cache=cache, + backend_id=backend_id, + model_id=model_id, + frame_size_seconds=runtime_config.pool_window_size_seconds, + frame_stride_seconds=runtime_config.pool_window_stride_seconds, + log_prefix="Readiness smoke round-trip", + logger=logger, + read_audio=_read_probe, + ) + cache_round_trip = cache_round_trip and np.array_equal( + encoded.embeddings, second.embeddings + ) + feature_dim = int(pooled_rows[0].shape[1]) + if any(int(block.shape[1]) != feature_dim for block in pooled_rows): + raise ValueError("Backend smoke feature dimensionality is inconsistent.") + expected_classifier_dim = int(backend_object.feature_dim) * 2 + if feature_dim != expected_classifier_dim: + raise ValueError( + "Backend smoke feature dimension violates classifier contract: " + f"expected={expected_classifier_dim} actual={feature_dim}." + ) + return SmokeResult( + attempted=len(samples), + succeeded=len(samples), + feature_dim=feature_dim, + cache_round_trip=cache_round_trip, + backend_id=backend_id, + model_id=model_id, + device=device, + dtype=dtype, + ) + + +@contextmanager +def _backend_smoke_deadline(seconds: float) -> Iterator[None]: + """Enforces a hard wall-clock deadline on the main-thread backend smoke phase.""" + if threading.current_thread() is not threading.main_thread(): + raise RuntimeError("Backend smoke requires main-thread hard-deadline support.") + if not hasattr(signal, "SIGALRM") or not hasattr(signal, "ITIMER_REAL"): + raise RuntimeError("Backend smoke hard-deadline support is unavailable on this platform.") + previous_handler = signal.getsignal(signal.SIGALRM) + + def _raise_timeout(_signum: int, _frame: object) -> None: + raise TimeoutError(f"Backend smoke exceeded hard wall-clock timeout ({seconds:.3f}s).") + + signal.signal(signal.SIGALRM, _raise_timeout) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + signal.setitimer(signal.ITIMER_REAL, 0.0) + signal.signal(signal.SIGALRM, previous_handler) + + +def run_selected_backend_smoke( + *, + settings: AppConfig, + samples: tuple[Utterance, ...], + probe_cache_dir: Path, +) -> SmokeResult: + """Runs selected-backend smoke under a configured hard wall-clock deadline.""" + raw_timeout = os.getenv("SER_TRAINING_SMOKE_TIMEOUT_SECONDS", "120") + try: + timeout_seconds = float(raw_timeout) + except ValueError as error: + raise ValueError("SER_TRAINING_SMOKE_TIMEOUT_SECONDS must be numeric.") from error + if not np.isfinite(timeout_seconds) or not 0.0 < timeout_seconds <= 600.0: + raise ValueError("Training smoke timeout must be finite and within (0, 600] seconds.") + with _backend_smoke_deadline(timeout_seconds): + return _run_selected_backend_smoke_unbounded( + settings=settings, + samples=samples, + probe_cache_dir=probe_cache_dir, + ) + + +def reuse_checked_backend( + *, + backend_id: str, + model_id: str, + device: str, + dtype: str, + build: Callable[[], FeatureBackend], +) -> FeatureBackend: + """Transfers the already checked backend when selectors match, otherwise rebuilds.""" + state = current_training_state() + if ( + state.checked_backend is not None + and state.checked_backend_id == backend_id + and state.checked_model_id == model_id + and state.checked_device == device + and state.checked_dtype == dtype + ): + return state.checked_backend + if state.checked_backend is not None: + close_checked_backend(state) + backend = build() + state.checked_backend = backend + state.checked_backend_id = backend_id + state.checked_model_id = model_id + state.checked_device = device + state.checked_dtype = dtype + return backend + + +def ensure_entrypoint_readiness( + *, + settings: AppConfig, + load_utterances: Callable[[], list[Utterance] | None], +) -> tuple[ReadinessReport, tuple[Utterance, ...]]: + """Runs mandatory readiness once for a profile entrypoint and caches its result.""" + state = current_training_state() + cache_state = state is not _DEFAULT_STATE + if cache_state and state.readiness is not None: + return state.readiness, state.utterances + smoke_runner = run_selected_backend_smoke + report, utterances = run_training_readiness( + settings=settings, + load_utterances=load_utterances, + smoke_runner=smoke_runner, + repair=state.operation.repair, + persist_quarantine_ledger=state.operation.mode is not TrainingMode.DRY_RUN, + ) + for phase in ( + "configuration", + "dataset_registry", + "dataset_media", + "split_feasibility", + "filesystem_resources", + "backend_smoke", + ): + phase_findings = [finding for finding in report.findings if finding.check == phase] + status = ( + "FAIL" + if any(finding.blocking for finding in phase_findings) + else "WARN" if phase_findings else "PASS" + ) + logger.info("CHECK %-25s %s", phase.replace("_", " "), status) + if not report.ready: + raise TrainingReadinessError(report) + logger.info( + "READY profile=%s plan=%s", + report.profile, + report.settings_digest[:12], + ) + if cache_state: + state.readiness = report + state.utterances = utterances + state.quarantine_records = list(report.quarantine) + state.cache_hits = 0 + state.cache_misses = 0 + state.recomputed_cache_entries = 0 + state.dropped_windows = 0 + state.medium_noise_stats_by_sample.clear() + return report, utterances + + +def record_cache_activity(*, cache_hit: bool, recomputed: bool) -> None: + """Records bounded cache activity for final artifact provenance.""" + state = current_training_state() + if cache_hit: + state.cache_hits += 1 + else: + state.cache_misses += 1 + if recomputed: + state.recomputed_cache_entries += 1 + state.containment_counts["cache:cache_corrupt:recompute"] += 1 + + +def record_optional_artifact_failure(error: Exception) -> None: + """Records an explicitly typed optional-artifact failure or aborts.""" + classification = classify_failure(error, scope=FailureScope.OPTIONAL_ARTIFACT) + if classification.disposition is not FailureDisposition.CONTINUE: + raise error + state = current_training_state() + state.containment_counts["optional_artifact:optional_artifact_failed:continue"] += 1 + logger.warning("Optional artifact failed: %s", classification.diagnostic) + + +def record_medium_noise_stats(*, sample_id: str, stats: object) -> None: + """Records per-sample medium noise statistics for canonical partition provenance.""" + if not sample_id: + raise ValueError("Medium noise statistics require a sample identity.") + if training_operation_active(): + current_training_state().medium_noise_stats_by_sample[sample_id] = stats + + +def bounded_retry_local_io( + operation: Callable[[], _PreparationT], + *, + identity: str, + max_retries: int = 2, + base_delay_seconds: float = 0.05, +) -> _PreparationT: + """Retries only typed transient local I/O with bounded deterministic jitter.""" + if max_retries < 0 or base_delay_seconds < 0.0: + raise ValueError("Retry bounds must be non-negative.") + for attempt in range(max_retries + 1): + try: + return operation() + except OSError as error: + classification = classify_failure(error, scope=FailureScope.SAMPLE) + if ( + classification.disposition is not FailureDisposition.BOUNDED_RETRY + or attempt >= max_retries + ): + raise + state = current_training_state() + state.bounded_retries += 1 + state.containment_counts["sample:media_decode_failed:bounded_retry"] += 1 + jitter = 0.75 + (int(digest_payload(identity)[:8], 16) % 501) / 1000.0 + delay = base_delay_seconds * (2**attempt) * jitter + logger.warning( + "Retrying transient local I/O attempt=%d/%d delay=%.3fs identity=%s", + attempt + 1, + max_retries, + delay, + identity[:80], + ) + time.sleep(delay) + raise AssertionError("bounded retry loop exhausted without returning or raising") + + +def record_preparation_progress(*, processed: int, total: int, sample_id: str) -> None: + """Logs bounded deterministic preparation progress and a credible linear ETA.""" + if processed <= 0 or total <= 0 or processed > total: + raise ValueError("Preparation progress must satisfy 0 < processed <= total.") + state = current_training_state() + now = time.monotonic() + if state.preparation_started_at is None or processed == 1: + state.preparation_started_at = now + interval = max(1, total // 10) + if processed != 1 and processed != total and processed % interval: + return + elapsed = max(0.0, now - state.preparation_started_at) + eta: float | None = None + if processed >= 2 and elapsed > 0.0: + eta = elapsed * (total - processed) / processed + logger.info( + "PREPARE processed=%d total=%d cache_hits=%d cache_misses=%d recomputed=%d " + "quarantined=%d elapsed=%.1fs eta=%s sample=%s", + processed, + total, + state.cache_hits, + state.cache_misses, + state.recomputed_cache_entries, + len(state.quarantine_records), + elapsed, + f"{eta:.1f}s" if eta is not None else "unknown", + sample_id[:80], + ) + + +def record_dropped_windows(count: int) -> None: + """Adds a non-negative dropped-window count to invocation provenance.""" + if count < 0: + raise ValueError("Dropped-window count must be non-negative.") + state = current_training_state() + state.dropped_windows += count + state.containment_counts["window:window_low_variance:continue"] += count + + +def build_training_robustness_provenance() -> dict[str, object]: + """Builds the durable readiness/quarantine/cache provenance payload.""" + state = current_training_state() + readiness = state.readiness + return { + "readiness": ( + { + "schema_version": readiness.schema_version, + "settings_digest": readiness.settings_digest, + "registry_digest": readiness.registry_digest, + "manifest_digest": readiness.manifest_digest, + "media_digest": readiness.media_digest, + } + if readiness is not None + else None + ), + "quarantine": [record.to_dict() for record in state.quarantine_records], + "statistics": { + "quarantined_samples": len(state.quarantine_records), + "dropped_windows": state.dropped_windows, + "cache_hits": state.cache_hits, + "cache_misses": state.cache_misses, + "recomputed_cache_entries": state.recomputed_cache_entries, + "bounded_retries": state.bounded_retries, + "containment": dict(sorted(state.containment_counts.items())), + }, + } + + +def handle_sample_encoding_failure( + *, + settings: AppConfig, + sample: Utterance, + error: Exception, +) -> bool: + """Quarantines one proven local encoding failure only when every budget permits.""" + classification = classify_failure( + error, + scope=FailureScope.SAMPLE, + sample=sample, + allowed_roots=_allowed_media_roots(settings), + ) + if classification.disposition is not FailureDisposition.QUARANTINE: + return False + state = current_training_state() + enforce_quarantine_budget( + policy=QuarantinePolicy.from_settings(settings), + all_samples=state.utterances, + existing_records=state.quarantine_records, + candidate=sample, + classification=classification, + ) + from ser._internal.models.dataset_splitting import ( # noqa: TID251 + split_utterances_three_way, + ) + + projected_samples = [ + item + for item in state.utterances + if item.sample_id != sample.sample_id + and all(record.sample_id != item.sample_id for record in state.quarantine_records) + ] + try: + train_samples, dev_samples, test_samples, split_metadata = split_utterances_three_way( + samples=projected_samples, + settings=settings, + logger=logger, + ) + except (RuntimeError, ValueError) as split_error: + raise QuarantineBudgetExceeded( + f"Projected canonical split is infeasible: {split_error}" + ) from split_error + required_classes = {item.require_label() for item in projected_samples} + minimum = settings.data_loader.min_remaining_per_class_split + for partition_name, partition in ( + ("train", train_samples), + ("dev", dev_samples), + ("test", test_samples), + ): + support = { + label: sum(item.label == label for item in partition) for label in required_classes + } + if any(count < minimum for count in support.values()): + raise QuarantineBudgetExceeded( + f"Projected {partition_name} split violates class support: {support}." + ) + if split_metadata.speaker_overlap_count > 0: + raise QuarantineBudgetExceeded("Projected quarantine introduces speaker leakage.") + timestamp = datetime.now(UTC).isoformat() + record = build_quarantine_record( + sample=sample, + classification=classification, + occurred_at=timestamp, + retry_count=0, + ) + state.quarantine_records.append(record) + state.containment_counts[f"sample:{classification.reason_code.value}:quarantine"] += 1 + state.utterances = tuple( + item for item in state.utterances if item.sample_id != sample.sample_id + ) + state.preparation_quarantine_changed = True + ledger_path = settings.tmp_folder / f"quarantine-{resolve_profile_name(settings)}.jsonl" + write_quarantine_ledger(ledger_path, state.quarantine_records) + logger.warning( + "Quarantined sample %s (corpus=%s reason=%s).", + sample.sample_id, + sample.corpus, + classification.reason_code.value, + ) + return True + + +def prepare_until_quarantine_stable( + *, + settings: AppConfig, + prepare: Callable[[], _PreparationT], +) -> _PreparationT: + """Repeats split/preparation whenever quarantine changes its effective inputs.""" + state = current_training_state() + data_loader_config = getattr(settings, "data_loader", None) + configured_max = getattr(data_loader_config, "max_failed_files", 25) + max_failed_files = ( + configured_max + if isinstance(configured_max, int) and not isinstance(configured_max, bool) + else 25 + ) + max_passes = max_failed_files + 1 + for _ in range(max_passes): + state.preparation_quarantine_changed = False + prepared = prepare() + if not state.preparation_quarantine_changed: + return prepared + raise RuntimeError("Preparation quarantine did not stabilize within its absolute budget.") + + +def prepared_plan_for_operation(settings: AppConfig) -> PreparedPlan | None: + """Loads the explicit prepared plan requested by the active real-training operation.""" + path = current_training_state().operation.prepared_plan + return load_prepared_plan(path) if path is not None else None + + +def serialize_metadata(value: object) -> object: + """Converts prepared DTO metadata into bounded canonical JSON values.""" + if is_dataclass(value) and not isinstance(value, type): + return serialize_metadata(asdict(value)) + if isinstance(value, Path): + return str(value) + if isinstance(value, Mapping): + return {str(key): serialize_metadata(item) for key, item in value.items()} + if isinstance(value, tuple | list): + return [serialize_metadata(item) for item in value] + if value is None or isinstance(value, str | int | float | bool): + return value + raise TypeError(f"Unsupported prepared metadata type: {type(value).__name__}") + + +def write_prepared_feature_payload( + *, + settings: AppConfig, + x_train: NDArray[np.float64], + x_dev: NDArray[np.float64], + x_test: NDArray[np.float64], + y_train: Sequence[str], + y_dev: Sequence[str], + y_test: Sequence[str], + metadata: Mapping[str, object], + path: Path | None = None, +) -> Path: + """Atomically persists reusable features without pickle/object arrays.""" + destination = path or default_prepared_payload_path(settings) + destination.parent.mkdir(parents=True, exist_ok=True) + descriptor, raw_tmp_path = tempfile.mkstemp( + prefix=f".{destination.name}.", dir=destination.parent + ) + tmp_path = Path(raw_tmp_path) + try: + with os.fdopen(descriptor, "wb") as handle: + np.savez_compressed( + handle, + x_train=np.asarray(x_train, dtype=np.float64), + x_dev=np.asarray(x_dev, dtype=np.float64), + x_test=np.asarray(x_test, dtype=np.float64), + y_train=np.asarray(list(y_train), dtype=np.str_), + y_dev=np.asarray(list(y_dev), dtype=np.str_), + y_test=np.asarray(list(y_test), dtype=np.str_), + metadata_json=np.asarray( + json.dumps(serialize_metadata(metadata), sort_keys=True, separators=(",", ":")), + dtype=np.str_, + ), + ) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, destination) + finally: + tmp_path.unlink(missing_ok=True) + return destination + + +def read_prepared_feature_payload(plan: PreparedPlan) -> PreparedFeaturePayload: + """Reads and semantically validates one digest-protected prepared payload.""" + payload_path = Path(plan.payload_path) + try: + if hash_file(payload_path) != plan.payload_digest: + raise PreparedPlanError("Prepared feature payload digest changed after plan loading.") + with np.load(payload_path, allow_pickle=False) as payload: + required = { + "x_train", + "x_dev", + "x_test", + "y_train", + "y_dev", + "y_test", + "metadata_json", + } + if set(payload.files) != required: + raise PreparedPlanError("Prepared feature payload keys are incomplete or unknown.") + raw_x_train = np.asarray(payload["x_train"]) + raw_x_dev = np.asarray(payload["x_dev"]) + raw_x_test = np.asarray(payload["x_test"]) + raw_y_train = np.asarray(payload["y_train"]) + raw_y_dev = np.asarray(payload["y_dev"]) + raw_y_test = np.asarray(payload["y_test"]) + raw_metadata = np.asarray(payload["metadata_json"]) + except PreparedPlanError: + raise + except (OSError, ValueError, KeyError) as error: + raise PreparedPlanError(f"Prepared feature payload is invalid: {error}") from error + if any( + str(matrix.dtype) != plan.feature_dtype for matrix in (raw_x_train, raw_x_dev, raw_x_test) + ): + raise PreparedPlanError("Prepared feature payload dtype differs from the plan.") + if any(matrix.ndim != 2 for matrix in (raw_x_train, raw_x_dev, raw_x_test)): + raise PreparedPlanError("Prepared feature matrices must all be two-dimensional.") + combined_shape = ( + int(raw_x_train.shape[0] + raw_x_dev.shape[0] + raw_x_test.shape[0]), + int(raw_x_train.shape[1]), + ) + if len({raw_x_train.shape[1], raw_x_dev.shape[1], raw_x_test.shape[1]}) != 1 or ( + combined_shape != plan.feature_shape + ): + raise PreparedPlanError("Prepared feature matrices differ from the planned shape.") + x_train = np.asarray(raw_x_train, dtype=np.float64) + x_dev = np.asarray(raw_x_dev, dtype=np.float64) + x_test = np.asarray(raw_x_test, dtype=np.float64) + if not all(np.all(np.isfinite(matrix)) for matrix in (x_train, x_dev, x_test)): + raise PreparedPlanError("Prepared feature matrices contain non-finite values.") + if any(labels.ndim != 1 for labels in (raw_y_train, raw_y_dev, raw_y_test)): + raise PreparedPlanError("Prepared labels must be one-dimensional arrays.") + y_train = [str(item) for item in raw_y_train] + y_dev = [str(item) for item in raw_y_dev] + y_test = [str(item) for item in raw_y_test] + if any(not label.strip() for label in (*y_train, *y_dev, *y_test)): + raise PreparedPlanError("Prepared labels must be non-empty strings.") + if ( + x_train.shape[0] != len(y_train) + or x_dev.shape[0] != len(y_dev) + or x_test.shape[0] != len(y_test) + ): + raise PreparedPlanError("Prepared feature/label row counts do not match.") + if raw_metadata.ndim != 0: + raise PreparedPlanError("Prepared feature metadata payload must be scalar JSON.") + try: + parsed = json.loads(str(raw_metadata.item())) + except json.JSONDecodeError as error: + raise PreparedPlanError("Prepared feature metadata is invalid JSON.") from error + if not isinstance(parsed, dict): + raise PreparedPlanError("Prepared feature metadata must be a JSON object.") + for key, expected in ( + ("sample_ledger", list(plan.sample_ledger)), + ("window_ledger", list(plan.window_ledger)), + ("disposition_counts", dict(plan.disposition_counts)), + ): + if parsed.get(key) != serialize_metadata(expected): + raise PreparedPlanError(f"Prepared feature metadata {key} differs from the plan.") + included_ids = { + str(row["sample_id"]) + for row in plan.sample_ledger + if row.get("disposition") == "included" and isinstance(row.get("sample_id"), str) + } + current_ids = {item.sample_id for item in _effective_state_utterances()} + if included_ids != current_ids: + raise PreparedPlanError("Prepared sample ledger differs from current effective utterances.") + planned_labels = [ + str(row["class"]) for row in plan.window_ledger if row.get("disposition") == "included" + ] + if planned_labels != [*y_train, *y_dev, *y_test]: + raise PreparedPlanError("Prepared window ledger labels differ from payload labels.") + return PreparedFeaturePayload(x_train, x_dev, x_test, y_train, y_dev, y_test, parsed) + + +def active_plan_path(settings: AppConfig) -> Path: + """Returns the explicit plan path or the deterministic default destination.""" + return current_training_state().operation.prepared_plan or default_prepared_plan_path(settings) + + +def canonical_train_rows( + *, + settings: AppConfig, + x_train: NDArray[np.float64], + y_train: Sequence[str], + train_sample_ids: Sequence[str], +) -> tuple[NDArray[np.float64], list[str]]: + """Filters fresh two-way preparation to the canonical three-way train subset.""" + filtered_x, filtered_y, _, _ = canonical_train_partition( + settings=settings, + x_train=x_train, + y_train=y_train, + train_metadata=train_sample_ids, + sample_id=lambda item: item, + ) + return filtered_x, filtered_y + + +def canonical_train_partition( + *, + settings: AppConfig, + x_train: NDArray[np.float64], + y_train: Sequence[str], + train_metadata: Sequence[_PreparationT], + sample_id: Callable[[_PreparationT], str], +) -> tuple[NDArray[np.float64], list[str], list[_PreparationT], list[Utterance]]: + """Returns canonical train rows, metadata, and utterances from fresh preparation.""" + from ser._internal.models.dataset_splitting import ( # noqa: TID251 + split_utterances_three_way, + ) + + train, _, _, _ = split_utterances_three_way( + samples=list(_effective_state_utterances()), settings=settings, logger=logger + ) + allowed = {item.sample_id for item in train} + identifiers = [sample_id(item) for item in train_metadata] + indices = [index for index, identifier in enumerate(identifiers) if identifier in allowed] + if len(train_metadata) != len(y_train) or len(y_train) != int(x_train.shape[0]): + raise PreparedPlanError("Fresh training rows lack exact per-window sample identities.") + return ( + np.asarray(x_train[indices], dtype=np.float64), + [y_train[index] for index in indices], + [train_metadata[index] for index in indices], + train, + ) + + +def training_meta_sample_ids(metadata: Sequence[object]) -> list[str]: + """Extracts validated sample identities from typed or mapping window metadata.""" + identifiers: list[str] = [] + for item in metadata: + raw = ( + item.get("sample_id") if isinstance(item, Mapping) else getattr(item, "sample_id", None) + ) + if not isinstance(raw, str) or not raw: + raise PreparedPlanError("Training window metadata lacks a valid sample_id.") + identifiers.append(raw) + return identifiers + + +def _prepared_ledgers( + *, + settings: AppConfig, + y_train: Sequence[str], + y_dev: Sequence[str], + y_test: Sequence[str], + metadata: Mapping[str, object], +) -> tuple[ + tuple[Mapping[str, object], ...], + tuple[Mapping[str, object], ...], + dict[str, int], + dict[str, Mapping[str, int]], +]: + """Builds final actual split/window/disposition ledgers for one prepared payload.""" + from ser._internal.models.dataset_splitting import ( # noqa: TID251 + split_utterances_three_way, + ) + + effective = list(_effective_state_utterances()) + train, dev, test, _ = split_utterances_three_way( + samples=effective, settings=settings, logger=logger + ) + train_ids = [item.sample_id for item in train] + dev_ids = [item.sample_id for item in dev] + test_ids = [item.sample_id for item in test] + declared_train_ids = metadata.get("train_sample_ids") + declared_test_ids = metadata.get("test_sample_ids") + declared_dev_ids = metadata.get("dev_sample_ids") + if ( + declared_train_ids is not None + and list(cast(Sequence[str], declared_train_ids)) != train_ids + ): + raise PreparedPlanError("Prepared train sample ledger differs from the current split.") + if declared_test_ids is not None and list(cast(Sequence[str], declared_test_ids)) != test_ids: + raise PreparedPlanError("Prepared test sample ledger differs from the current split.") + if declared_dev_ids is not None and list(cast(Sequence[str], declared_dev_ids)) != dev_ids: + raise PreparedPlanError("Prepared dev sample ledger differs from the current split.") + sample_by_id = {item.sample_id: item for item in effective} + sample_rows: list[Mapping[str, object]] = [ + { + "sample_id": sample_id, + "partition": partition, + "class": sample_by_id[sample_id].require_label(), + "disposition": "included", + } + for partition, identifiers in (("train", train_ids), ("dev", dev_ids), ("test", test_ids)) + for sample_id in identifiers + ] + sample_rows.extend( + { + "sample_id": record.sample_id, + "partition": record.split, + "class": record.primary_class, + "disposition": "quarantined", + "reason_code": record.reason_code.value, + } + for record in current_training_state().quarantine_records + ) + + def _window_rows( + partition: str, + labels: Sequence[str], + sample_ids: Sequence[str], + raw_meta: object, + ) -> list[Mapping[str, object]]: + if raw_meta is None: + if len(labels) != len(sample_ids): + raise PreparedPlanError( + f"Prepared {partition} rows require an explicit per-window sample ledger." + ) + meta_rows: list[Mapping[str, object]] = [ + {"sample_id": sample_id} for sample_id in sample_ids + ] + else: + normalized = serialize_metadata(raw_meta) + if not isinstance(normalized, list) or not all( + isinstance(item, dict) for item in normalized + ): + raise PreparedPlanError(f"Prepared {partition} window metadata is invalid.") + meta_rows = cast(list[Mapping[str, object]], normalized) + if len(meta_rows) != len(labels): + raise PreparedPlanError(f"Prepared {partition} window ledger count is stale.") + rows: list[Mapping[str, object]] = [] + allowed_ids = set(sample_ids) + for index, (label, meta) in enumerate(zip(labels, meta_rows, strict=True)): + sample_id = meta.get("sample_id") + if not isinstance(sample_id, str) or sample_id not in allowed_ids: + raise PreparedPlanError( + f"Prepared {partition} window references an unknown sample." + ) + rows.append( + { + "window_id": f"{partition}:{index}", + "sample_id": sample_id, + "partition": partition, + "class": label, + "disposition": "included", + } + ) + return rows + + window_rows = _window_rows("train", y_train, train_ids, metadata.get("train_meta")) + window_rows.extend(_window_rows("dev", y_dev, dev_ids, metadata.get("dev_meta"))) + window_rows.extend(_window_rows("test", y_test, test_ids, metadata.get("test_meta"))) + dropped = current_training_state().dropped_windows + if dropped: + window_rows.append({"window_id": "dropped", "disposition": "dropped", "count": dropped}) + dispositions = { + "included_samples": len(effective), + "quarantined_samples": len(current_training_state().quarantine_records), + "included_windows": len(y_train) + len(y_dev) + len(y_test), + "dropped_windows": dropped, + } + counts: dict[str, Mapping[str, int]] = { + "partition_samples": {"train": len(train_ids), "test": len(test_ids), "dev": len(dev_ids)}, + "partition_windows": {"train": len(y_train), "test": len(y_test), "dev": len(y_dev)}, + "class": dict(Counter(item.require_label() for item in effective)), + "corpus": dict(Counter(item.corpus for item in effective)), + "language": dict(Counter(item.language or "unknown" for item in effective)), + "native_split": dict(Counter(str(item.split or "unspecified") for item in effective)), + "partition_support": {"train": 1, "test": 1, "dev": 1}, + "disposition": dispositions, + } + return tuple(sample_rows), tuple(window_rows), dispositions, counts + + +def publish_prepared_features( + *, + settings: AppConfig, + backend_id: str, + model_id: str, + device: str, + dtype: str, + utterances: Sequence[Utterance], + x_train: NDArray[np.float64], + x_test: NDArray[np.float64], + y_train: Sequence[str], + y_test: Sequence[str], + metadata: Mapping[str, object], + cache_namespace: str, + windowing_policy: Mapping[str, object], + noise_statistics: Mapping[str, object], +) -> PreparedPlan: + """Atomically publishes safe reusable features followed by their ready plan.""" + started_at = time.perf_counter() + logger.info( + "PREPARED_PUBLISH_START backend_id=%s model_id=%s cache_namespace=%s rows=%d", + backend_id, + model_id, + cache_namespace, + int(x_train.shape[0] + x_test.shape[0]), + ) + state = current_training_state() + readiness = state.readiness + if readiness is None: + raise RuntimeError("Prepared publication requires a completed readiness pass.") + trusted_namespace = expected_cache_namespace(settings) + if cache_namespace != trusted_namespace: + raise PreparedPlanError( + f"Prepared cache namespace must be {trusted_namespace!r} for the active profile." + ) + validated_cache_root(settings, trusted_namespace) + from ser._internal.models.dataset_splitting import ( # noqa: TID251 + split_utterances, + split_utterances_three_way, + ) + + effective = list(_effective_state_utterances()) + initial_train, _, _ = split_utterances(samples=effective, settings=settings, logger=logger) + train_partition, dev_partition, test_partition, _ = split_utterances_three_way( + samples=effective, settings=settings, logger=logger + ) + train_ids = [item.sample_id for item in train_partition] + dev_ids = [item.sample_id for item in dev_partition] + test_ids = [item.sample_id for item in test_partition] + required_classes = {item.require_label() for item in effective} + minimum = settings.data_loader.min_remaining_per_class_split + for partition_name, partition in ( + ("train", train_partition), + ("dev", dev_partition), + ("test", test_partition), + ): + support = { + label: sum(item.label == label for item in partition) for label in required_classes + } + if any(count < minimum for count in support.values()): + raise PreparedPlanError( + f"Prepared {partition_name} split violates minimum class support: {support}." + ) + raw_train_meta = metadata.get("train_meta") + if raw_train_meta is None: + if len(y_train) != len(initial_train): + raise PreparedPlanError( + "Prepared train rows require per-window metadata for dev split." + ) + train_meta_rows: list[Mapping[str, object]] = [ + {"sample_id": item.sample_id} for item in initial_train + ] + else: + normalized_train_meta = serialize_metadata(raw_train_meta) + if not isinstance(normalized_train_meta, list) or not all( + isinstance(item, dict) for item in normalized_train_meta + ): + raise PreparedPlanError("Prepared train window metadata is invalid.") + train_meta_rows = cast(list[Mapping[str, object]], normalized_train_meta) + if len(train_meta_rows) != len(y_train) or len(y_train) != int(x_train.shape[0]): + raise PreparedPlanError("Prepared train feature/window metadata counts differ.") + train_id_set = set(train_ids) + dev_id_set = set(dev_ids) + retained_indices = [ + index for index, row in enumerate(train_meta_rows) if row.get("sample_id") in train_id_set + ] + dev_indices = [ + index for index, row in enumerate(train_meta_rows) if row.get("sample_id") in dev_id_set + ] + if len(retained_indices) + len(dev_indices) != len(train_meta_rows): + raise PreparedPlanError("Prepared train windows do not map to train/dev samples.") + x_dev = np.asarray(x_train[dev_indices], dtype=np.float64) + y_dev = [y_train[index] for index in dev_indices] + x_train = np.asarray(x_train[retained_indices], dtype=np.float64) + y_train = [y_train[index] for index in retained_indices] + metadata = { + **metadata, + "train_sample_ids": train_ids, + "dev_sample_ids": dev_ids, + "test_sample_ids": test_ids, + "train_meta": [train_meta_rows[index] for index in retained_indices], + "dev_meta": [train_meta_rows[index] for index in dev_indices], + } + effective_utterances = _effective_state_utterances() + sample_ledger, window_ledger, dispositions, final_counts = _prepared_ledgers( + settings=settings, + y_train=y_train, + y_dev=y_dev, + y_test=y_test, + metadata=metadata, + ) + payload_metadata = { + **metadata, + "quarantine": [record.to_dict() for record in state.quarantine_records], + "effective_sample_ids": [item.sample_id for item in effective_utterances], + "sample_ledger": list(sample_ledger), + "window_ledger": list(window_ledger), + "disposition_counts": dispositions, + } + payload_path = write_prepared_feature_payload( + settings=settings, + x_train=x_train, + x_dev=x_dev, + x_test=x_test, + y_train=y_train, + y_dev=y_dev, + y_test=y_test, + metadata=payload_metadata, + ) + cache_keys = _cache_keys(settings, cache_namespace) + recipe_digest = _recipe_digest(settings) + split_digest = _current_split_digest(settings) + quarantine_digest = _current_quarantine_digest() + model_revision = _resolved_model_revision(backend_id=backend_id, model_id=model_id) + plan = build_prepared_plan( + settings=settings, + readiness=readiness, + backend_id=backend_id, + model_id=model_id, + model_revision=model_revision, + device=device, + dtype=dtype, + recipe_digest=recipe_digest, + split_ledger_digest=split_digest, + quarantine_ledger_digest=quarantine_digest, + cache_namespace=trusted_namespace, + cache_version=PREPARED_CACHE_VERSION, + cache_keys=cache_keys, + effective_counts=final_counts, + sample_ledger=sample_ledger, + window_ledger=window_ledger, + disposition_counts=dispositions, + feature_shape=( + int(x_train.shape[0] + x_dev.shape[0] + x_test.shape[0]), + int(x_train.shape[1]), + ), + feature_dtype=str(np.asarray(x_train).dtype), + windowing_policy=windowing_policy, + noise_statistics=noise_statistics, + payload_path=payload_path, + package_version=_package_version(), + ) + write_prepared_plan(default_prepared_plan_path(settings), plan) + logger.info( + "PREPARED_PUBLISH_DONE payload_path=%s plan_path=%s feature_shape=%s elapsed=%.1fs", + payload_path, + default_prepared_plan_path(settings), + plan.feature_shape, + time.perf_counter() - started_at, + ) + return plan + + +def validate_operation_plan( + *, + settings: AppConfig, + backend_id: str, + model_id: str, + device: str, + dtype: str, +) -> PreparedPlan | None: + """Loads and validates an explicit plan against the current readiness snapshot.""" + plan = prepared_plan_for_operation(settings) + if plan is None: + return None + readiness = current_training_state().readiness + if readiness is None: + raise RuntimeError("Prepared plan validation requires a completed readiness pass.") + trusted_namespace = expected_cache_namespace(settings) + validated_cache_root(settings, plan.cache_namespace) + model_revision = _resolved_model_revision(backend_id=backend_id, model_id=model_id) + validate_prepared_plan( + plan, + settings=settings, + readiness=readiness, + backend_id=backend_id, + model_id=model_id, + model_revision=model_revision, + device=device, + dtype=dtype, + recipe_digest=_recipe_digest(settings), + split_ledger_digest=_current_split_digest(settings), + quarantine_ledger_digest=_current_quarantine_digest(), + cache_namespace=trusted_namespace, + cache_version=PREPARED_CACHE_VERSION, + cache_keys=_cache_keys(settings, trusted_namespace), + package_version=_package_version(), + ) + return plan + + +__all__ = [ + "PreparedFeaturePayload", + "TrainingRunState", + "active_plan_path", + "build_training_robustness_provenance", + "close_checked_backend", + "current_training_state", + "ensure_entrypoint_readiness", + "handle_sample_encoding_failure", + "prepared_plan_for_operation", + "prepare_until_quarantine_stable", + "publish_prepared_features", + "read_prepared_feature_payload", + "record_cache_activity", + "record_dropped_windows", + "reuse_checked_backend", + "run_selected_backend_smoke", + "serialize_metadata", + "training_operation_scope", + "training_operation_active", + "validate_operation_plan", + "write_prepared_feature_payload", +] diff --git a/ser/_internal/models/training_preparation.py b/ser/_internal/models/training_preparation.py index 8dc692d..2722f4d 100644 --- a/ser/_internal/models/training_preparation.py +++ b/ser/_internal/models/training_preparation.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging +import time from collections.abc import Callable, Sequence from typing import TypeVar @@ -114,23 +115,52 @@ def prepare_medium_training_features( ) -> MediumTrainingPreparation[_UtteranceT, _SplitMetaT, _MetaT, _NoiseStatsT]: """Builds medium-profile feature matrices and runtime metadata for training.""" + started_at = time.perf_counter() + logger = logging.getLogger(__name__) + logger.info("PREPARE_START stage=medium_features utterances=%d", len(utterances)) train_utterances, test_utterances, split_metadata = split_utterances(utterances) + logger.info( + "PREPARE_SPLIT_DONE train=%d test=%d elapsed=%.1fs", + len(train_utterances), + len(test_utterances), + time.perf_counter() - started_at, + ) model_id = resolve_model_id() runtime_device, runtime_dtype = resolve_runtime_selectors() + logger.info( + "PREPARE_BACKEND_START model_id=%s device=%s dtype=%s", + model_id, + runtime_device, + runtime_dtype, + ) backend = build_backend(model_id, runtime_device, runtime_dtype) cache = build_cache() - x_train, y_train, _train_meta, train_noise_stats = build_feature_dataset( + logger.info("PREPARE_FEATURES_START partition=train utterances=%d", len(train_utterances)) + x_train, y_train, train_meta, train_noise_stats = build_feature_dataset( train_utterances, backend, cache, model_id, ) + logger.info( + "PREPARE_FEATURES_DONE partition=train rows=%d labels=%d elapsed=%.1fs", + int(x_train.shape[0]), + len(y_train), + time.perf_counter() - started_at, + ) + logger.info("PREPARE_FEATURES_START partition=test utterances=%d", len(test_utterances)) x_test, y_test, test_meta, test_noise_stats = build_feature_dataset( test_utterances, backend, cache, model_id, ) + logger.info( + "PREPARE_DONE stage=medium_features train_rows=%d test_rows=%d elapsed=%.1fs", + int(x_train.shape[0]), + int(x_test.shape[0]), + time.perf_counter() - started_at, + ) return MediumTrainingPreparation( train_utterances=train_utterances, test_utterances=test_utterances, @@ -140,6 +170,7 @@ def prepare_medium_training_features( runtime_dtype=runtime_dtype, x_train=x_train, y_train=y_train, + train_meta=train_meta, x_test=x_test, y_test=y_test, test_meta=test_meta, @@ -166,23 +197,52 @@ def prepare_accurate_training_features( ) -> AccurateTrainingPreparation[_UtteranceT, _SplitMetaT, _MetaT]: """Builds accurate-profile feature matrices and runtime metadata for training.""" + started_at = time.perf_counter() + logger = logging.getLogger(__name__) + logger.info("PREPARE_START stage=accurate_features utterances=%d", len(utterances)) train_utterances, test_utterances, split_metadata = split_utterances(utterances) + logger.info( + "PREPARE_SPLIT_DONE train=%d test=%d elapsed=%.1fs", + len(train_utterances), + len(test_utterances), + time.perf_counter() - started_at, + ) model_id = resolve_model_id() runtime_device, runtime_dtype = resolve_runtime_selectors() + logger.info( + "PREPARE_BACKEND_START model_id=%s device=%s dtype=%s", + model_id, + runtime_device, + runtime_dtype, + ) backend = build_backend(model_id, runtime_device, runtime_dtype) cache = build_cache() - x_train, y_train, _train_meta = build_feature_dataset( + logger.info("PREPARE_FEATURES_START partition=train utterances=%d", len(train_utterances)) + x_train, y_train, train_meta = build_feature_dataset( train_utterances, backend, cache, model_id, ) + logger.info( + "PREPARE_FEATURES_DONE partition=train rows=%d labels=%d elapsed=%.1fs", + int(x_train.shape[0]), + len(y_train), + time.perf_counter() - started_at, + ) + logger.info("PREPARE_FEATURES_START partition=test utterances=%d", len(test_utterances)) x_test, y_test, test_meta = build_feature_dataset( test_utterances, backend, cache, model_id, ) + logger.info( + "PREPARE_DONE stage=accurate_features train_rows=%d test_rows=%d elapsed=%.1fs", + int(x_train.shape[0]), + int(x_test.shape[0]), + time.perf_counter() - started_at, + ) return AccurateTrainingPreparation( train_utterances=train_utterances, test_utterances=test_utterances, @@ -192,6 +252,7 @@ def prepare_accurate_training_features( runtime_dtype=runtime_dtype, x_train=x_train, y_train=y_train, + train_meta=train_meta, x_test=x_test, y_test=y_test, test_meta=test_meta, diff --git a/ser/_internal/models/training_types.py b/ser/_internal/models/training_types.py index 279d63a..d813679 100644 --- a/ser/_internal/models/training_types.py +++ b/ser/_internal/models/training_types.py @@ -117,6 +117,7 @@ class MediumTrainingPreparation(Generic[_UtteranceT, _SplitMetaT, _MetaT, _Noise runtime_dtype: str x_train: np.ndarray y_train: list[str] + train_meta: list[_MetaT] x_test: np.ndarray y_test: list[str] test_meta: list[_MetaT] @@ -136,6 +137,7 @@ class AccurateTrainingPreparation(Generic[_UtteranceT, _SplitMetaT, _MetaT]): runtime_dtype: str x_train: np.ndarray y_train: list[str] + train_meta: list[_MetaT] x_test: np.ndarray y_test: list[str] test_meta: list[_MetaT] diff --git a/tests/suites/integration/models/test_emotion_model_contracts.py b/tests/suites/integration/models/test_emotion_model_contracts.py index ea1c202..b196c5e 100644 --- a/tests/suites/integration/models/test_emotion_model_contracts.py +++ b/tests/suites/integration/models/test_emotion_model_contracts.py @@ -48,6 +48,7 @@ def test_train_model_raises_when_dataset_is_missing( monkeypatch: pytest.MonkeyPatch, ) -> None: """`train_model` should fail closed when no dataset split can be loaded.""" + _stub_training_readiness(monkeypatch) settings = SimpleNamespace( training=SimpleNamespace(test_size=0.25), models=SimpleNamespace( @@ -152,6 +153,7 @@ def test_training_entrypoints_train_model_builds_fast_hooks_from_canonical_owner monkeypatch: pytest.MonkeyPatch, ) -> None: """Fast training entrypoint should bind canonical collaborators directly.""" + _stub_training_readiness(monkeypatch) settings = cast(em.AppConfig, SimpleNamespace()) captured: dict[str, object] = {} @@ -175,9 +177,13 @@ def _fake_train_model(**kwargs: object) -> None: hooks = cast(FastTrainingHooks, captured["hooks"]) assert hooks.logger is training_entrypoints.logger assert hooks.settings is settings - assert isinstance(hooks.load_utterances, partial) - assert hooks.load_utterances.func is data_loader.load_utterances - assert hooks.load_utterances.keywords == {"settings": settings} + effective_utterance = object() + monkeypatch.setattr( + training_entrypoints._training_orchestration, + "current_training_state", + lambda: SimpleNamespace(utterances=(effective_utterance,)), + ) + assert hooks.load_utterances() == [effective_utterance] assert isinstance(hooks.ensure_dataset_consents_for_training, partial) assert ( hooks.ensure_dataset_consents_for_training.func @@ -214,6 +220,7 @@ def test_training_entrypoints_train_medium_model_uses_canonical_helpers( tmp_path: Path, ) -> None: """Medium training entrypoint should use direct owner modules, not emotion-model aliases.""" + _stub_training_readiness(monkeypatch) settings = cast( em.AppConfig, SimpleNamespace( @@ -240,10 +247,14 @@ def test_training_entrypoints_train_medium_model_uses_canonical_helpers( assert captured["settings"] is settings assert captured["logger"] is training_entrypoints.logger - load_utterances = captured["load_utterances_for_training"] - assert isinstance(load_utterances, partial) - assert load_utterances.func is data_loader.load_utterances - assert load_utterances.keywords == {"settings": settings} + effective_utterance = object() + monkeypatch.setattr( + training_entrypoints._training_orchestration, + "current_training_state", + lambda: SimpleNamespace(utterances=(effective_utterance,)), + ) + load_utterances = cast(Callable[[], list[object]], captured["load_utterances_for_training"]) + assert load_utterances() == [effective_utterance] ensure_consents = captured["ensure_dataset_consents_for_training"] assert isinstance(ensure_consents, partial) assert ensure_consents.func is training_support.ensure_dataset_consents_for_training @@ -299,6 +310,7 @@ def test_training_entrypoints_train_accurate_model_uses_canonical_helpers( tmp_path: Path, ) -> None: """Accurate training entrypoint should assemble direct owner collaborators.""" + _stub_training_readiness(monkeypatch) settings = cast( em.AppConfig, SimpleNamespace( @@ -352,10 +364,12 @@ def test_training_entrypoints_train_accurate_research_model_uses_canonical_helpe tmp_path: Path, ) -> None: """Research training entrypoint should use direct license and runtime owners.""" + events: list[str] = [] settings = cast( em.AppConfig, SimpleNamespace( models=SimpleNamespace( + folder=tmp_path / "models", huggingface_cache_root=tmp_path / "hf-cache", modelscope_cache_root=tmp_path / "ms-cache", ), @@ -370,6 +384,35 @@ def test_training_entrypoints_train_accurate_research_model_uses_canonical_helpe captured: dict[str, object] = {} sentinel_runner = object() + def _parse_access() -> frozenset[str]: + events.append("parse_access") + return frozenset() + + def _load_access(**_kwargs: object) -> frozenset[str]: + events.append("load_access") + return frozenset() + + monkeypatch.setattr( + license_check, + "parse_allowed_restricted_backends_env", + _parse_access, + ) + monkeypatch.setattr( + license_check, + "load_persisted_backend_consents", + _load_access, + ) + monkeypatch.setattr( + license_check, + "ensure_backend_access", + lambda **_kwargs: events.append("ensure_access"), + ) + monkeypatch.setattr( + training_entrypoints._training_orchestration, + "ensure_entrypoint_readiness", + lambda **_kwargs: events.append("readiness"), + ) + monkeypatch.setattr( accurate_training_preparation, "train_accurate_research_profile_model", @@ -391,6 +434,7 @@ def test_training_entrypoints_train_accurate_research_model_uses_canonical_helpe training_entrypoints.train_accurate_research_model(settings=settings) + assert events[:4] == ["parse_access", "load_access", "ensure_access", "readiness"] assert captured["settings"] is settings assert captured["logger"] is training_entrypoints.logger assert ( @@ -537,7 +581,9 @@ def test_predict_emotions_detailed_rejects_feature_size_mismatch( monkeypatch.setattr( em, "load_model", - lambda *, settings=None, expected_backend_id=None, expected_profile=None, expected_backend_model_id=None: loaded_model, + lambda *, settings=None, expected_backend_id=None, expected_profile=None, expected_backend_model_id=None: ( + loaded_model + ), ) with pytest.raises(ValueError, match="Feature vector size mismatch"): @@ -654,3 +700,12 @@ def test_predict_emotions_returns_legacy_segments_from_detailed_output( assert [segment.emotion for segment in result] == ["v1"] assert captured["file"] == "sample.wav" + + +def _stub_training_readiness(monkeypatch: pytest.MonkeyPatch) -> None: + """Keeps collaborator-wiring tests focused on their post-readiness seam.""" + monkeypatch.setattr( + training_entrypoints._training_orchestration, + "ensure_entrypoint_readiness", + lambda **_kwargs: None, + ) diff --git a/tests/suites/integration/models/test_training_preparation_and_execution.py b/tests/suites/integration/models/test_training_preparation_and_execution.py index 1c95f7b..d5b148f 100644 --- a/tests/suites/integration/models/test_training_preparation_and_execution.py +++ b/tests/suites/integration/models/test_training_preparation_and_execution.py @@ -192,6 +192,7 @@ def test_prepare_accurate_training_payload_builds_prepared_output() -> None: runtime_dtype="float32", x_train=np.asarray([[0.1, 0.2]], dtype=np.float64), y_train=["happy"], + train_meta=[{"sample_id": "u1"}], x_test=np.asarray([[0.3, 0.4]], dtype=np.float64), y_test=["sad"], test_meta=[{"sample_id": "u2"}], @@ -247,6 +248,7 @@ def test_prepare_medium_training_payload_builds_prepared_output() -> None: runtime_dtype="float32", x_train=np.asarray([[0.1, 0.2]], dtype=np.float64), y_train=["happy"], + train_meta=[{"sample_id": "u1"}], x_test=np.asarray([[0.3, 0.4]], dtype=np.float64), y_test=["sad"], test_meta=[{"sample_id": "u2"}], @@ -313,6 +315,10 @@ def predict(self, x_test: np.ndarray) -> Sequence[str]: runtime_dtype="float32", x_train=np.asarray([[0.1, 0.2], [0.3, 0.4]], dtype=np.float64), y_train=["happy", "sad"], + train_meta=[ + {"sample_id": "s1", "corpus": "ravdess", "language": "en"}, + {"sample_id": "s2", "corpus": "ravdess", "language": "en"}, + ], x_test=np.asarray([[0.5, 0.6]], dtype=np.float64), y_test=["happy"], test_meta=[{"sample_id": "s3", "corpus": "ravdess", "language": "en"}], @@ -434,6 +440,10 @@ def predict(self, x_test: np.ndarray) -> Sequence[str]: runtime_dtype="float32", x_train=np.asarray([[0.1, 0.2], [0.3, 0.4]], dtype=np.float64), y_train=["calm", "happy"], + train_meta=[ + {"sample_id": "s1", "corpus": "ravdess", "language": "en"}, + {"sample_id": "s2", "corpus": "ravdess", "language": "en"}, + ], x_test=np.asarray([[0.5, 0.6]], dtype=np.float64), y_test=["calm"], test_meta=[{"sample_id": "s3", "corpus": "ravdess", "language": "en"}], diff --git a/tests/suites/unit/data/test_data_loader.py b/tests/suites/unit/data/test_data_loader.py index ec27508..623f8cb 100644 --- a/tests/suites/unit/data/test_data_loader.py +++ b/tests/suites/unit/data/test_data_loader.py @@ -230,6 +230,15 @@ def test_load_utterances_prefers_manifest_when_configured( assert all(item.audio_path.is_absolute() for item in utterances) +def test_strict_dataset_audit_requires_explicit_recipe() -> None: + """The strict safety flag cannot silently become a no-op.""" + settings = _build_settings(max_failed_file_ratio=1.0) + settings.dataset.strict_audit = True + + with pytest.raises(RuntimeError, match="requires an explicit versioned dataset recipe"): + dl.load_utterances(settings=cast(AppConfig, settings)) + + def test_load_utterances_rejects_duplicate_sample_ids_across_manifests( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, @@ -308,6 +317,49 @@ def test_load_utterances_registry_uses_dataset_root_as_manifest_base_dir( assert utterances[1].audio_path == dataset_root / "clips" / "b.wav" +def test_load_utterances_rejects_registered_crema_pointer_before_manifest_load( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Existing CREMA-D manifests must not bypass media validation.""" + dataset_root = tmp_path / "datasets" / "crema-d" + audio_path = dataset_root / "AudioWAV" / "1077_WSI_ANG_XX.wav" + audio_path.parent.mkdir(parents=True) + audio_path.write_bytes( + b"version https://git-lfs.github.com/spec/v1\n" + b"oid sha256:be5c849653d28aaed49fbca687812f5352f295b1e1a66c269e4a3f2b7ae46489\n" + b"size 85462\n" + ) + manifest_path = tmp_path / "manifests" / "crema-d.jsonl" + manifest_path.parent.mkdir(parents=True) + manifest_path.write_text("{}\n", encoding="utf-8") + settings = _build_settings(max_failed_file_ratio=1.0) + settings.dataset.manifest_paths = () + monkeypatch.setattr( + dl, + "load_dataset_registry", + lambda settings: { + "crema-d": SimpleNamespace( + dataset_id="crema-d", + dataset_root=dataset_root, + manifest_path=manifest_path, + options={}, + ) + }, + ) + monkeypatch.setattr( + dl, + "load_manifest_jsonl", + lambda *_args, **_kwargs: pytest.fail("invalid manifest must not be loaded"), + ) + + with pytest.raises( + RuntimeError, + match=r"Registered CREMA-D dataset is not ready.*ser data download --dataset crema-d", + ): + dl.load_utterances(settings=cast(AppConfig, settings)) + + def test_load_utterances_registry_rebuilds_missing_manifest( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/suites/unit/models/test_accurate_training_artifact.py b/tests/suites/unit/models/test_accurate_training_artifact.py index 3a1895f..8259004 100644 --- a/tests/suites/unit/models/test_accurate_training_artifact.py +++ b/tests/suites/unit/models/test_accurate_training_artifact.py @@ -2,9 +2,9 @@ from __future__ import annotations +from collections.abc import Callable +from dataclasses import replace from pathlib import Path -from types import SimpleNamespace -from typing import cast import numpy as np import pytest @@ -17,6 +17,14 @@ import ser._internal.models.profile_runtime as profile_runtime import ser._internal.models.training_entrypoints as training_entrypoints import ser._internal.models.training_support as training_support +from ser._internal.config.schema import ( + AccurateResearchRuntimeConfig, + AccurateRuntimeConfig, + DatasetConfig, + MediumRuntimeConfig, + ModelsConfig, + RuntimeFlags, +) from ser._internal.data import EmbeddingCache from ser._internal.data.manifest import MANIFEST_SCHEMA_VERSION, Utterance from ser._internal.models import emotion_model as em @@ -26,40 +34,59 @@ from ser.config import AppConfig -def _settings_stub(tmp_path: Path) -> SimpleNamespace: - """Builds minimal settings stub required by accurate training helpers.""" - return SimpleNamespace( - dataset=SimpleNamespace( - glob_pattern=str(tmp_path / "*.wav"), - manifest_paths=(), - ), +def _settings_stub(tmp_path: Path) -> AppConfig: + """Builds a complete settings value required by accurate training.""" + return AppConfig( + emotions={"03": "happy", "04": "sad"}, + dataset=DatasetConfig(folder=tmp_path, subfolder_prefix=""), tmp_folder=tmp_path / "tmp", - models=SimpleNamespace( + models=ModelsConfig( folder=tmp_path / "models", - training_report_file=tmp_path / "training_report_accurate.json", + model_cache_dir=tmp_path / "model-cache", + training_report_file_name="training_report_accurate.json", accurate_model_id="openai/whisper-large-v3", accurate_research_model_id="iic/emotion2vec_plus_large", - huggingface_cache_root=tmp_path / "model-cache" / "huggingface", - modelscope_cache_root=tmp_path / "model-cache" / "modelscope" / "hub", ), - runtime_flags=SimpleNamespace(restricted_backends=True), - medium_runtime=SimpleNamespace( + runtime_flags=RuntimeFlags(restricted_backends=True), + medium_runtime=MediumRuntimeConfig( pool_window_size_seconds=1.0, pool_window_stride_seconds=1.0, ), - accurate_runtime=SimpleNamespace( + accurate_runtime=AccurateRuntimeConfig( pool_window_size_seconds=2.0, pool_window_stride_seconds=0.5, ), - accurate_research_runtime=SimpleNamespace( + accurate_research_runtime=AccurateResearchRuntimeConfig( pool_window_size_seconds=2.5, pool_window_stride_seconds=0.75, ), - torch_runtime=SimpleNamespace(device="auto", dtype="auto"), - feature_runtime_policy=SimpleNamespace(for_backend=lambda _backend_id: None), ) +def _stub_training_readiness(monkeypatch: pytest.MonkeyPatch) -> list[AppConfig]: + """Keeps artifact tests focused while preserving the checked-loader contract.""" + checked_settings: list[AppConfig] = [] + + def _ensure_entrypoint_readiness( + *, + settings: AppConfig, + load_utterances: Callable[[], list[Utterance] | None], + ) -> None: + checked_settings.append(settings) + utterances = load_utterances() + if utterances is None: + raise RuntimeError("Dataset not loaded") + state = training_entrypoints._training_orchestration.current_training_state() + state.utterances = tuple(utterances) + + monkeypatch.setattr( + training_entrypoints._training_orchestration, + "ensure_entrypoint_readiness", + _ensure_entrypoint_readiness, + ) + return checked_settings + + def _utterances_from_samples(samples: list[tuple[str, str]]) -> list[Utterance]: utterances: list[Utterance] = [] for index, (audio_path, label) in enumerate(samples): @@ -92,15 +119,18 @@ def test_train_accurate_model_requires_labeled_dataset( tmp_path: Path, ) -> None: """Accurate training should fail fast when dataset loading returns no samples.""" - monkeypatch.setattr(em, "reload_settings", lambda: _settings_stub(tmp_path)) + settings = _settings_stub(tmp_path) + checked_settings = _stub_training_readiness(monkeypatch) + monkeypatch.setattr(em, "reload_settings", lambda: settings) monkeypatch.setattr( training_entrypoints._data_loader, "load_utterances", - lambda *, settings=None: None, + lambda *, settings=None, allow_prepare=False: None, ) with pytest.raises(RuntimeError, match="Dataset not loaded"): em.train_accurate_model() + assert checked_settings == [settings] def test_train_accurate_model_persists_whisper_profile_metadata( @@ -142,11 +172,12 @@ def test_train_accurate_model_persists_whisper_profile_metadata( train_utterances = all_utterances[: len(train_samples)] test_utterances = all_utterances[len(train_samples) :] + _stub_training_readiness(monkeypatch) monkeypatch.setattr(em, "reload_settings", lambda: settings) monkeypatch.setattr( training_entrypoints._data_loader, "load_utterances", - lambda *, settings=None: all_utterances, + lambda *, settings=None, allow_prepare=False: all_utterances, ) monkeypatch.setattr( training_support, @@ -285,7 +316,10 @@ def test_train_accurate_model_uses_configured_model_id( ) -> None: """Accurate training should initialize backend/dataset with configured model id.""" settings = _settings_stub(tmp_path) - settings.models.accurate_model_id = "unit-test/whisper-tiny" + settings = replace( + settings, + models=replace(settings.models, accurate_model_id="unit-test/whisper-tiny"), + ) train_samples = [("train_0.wav", "happy"), ("train_1.wav", "sad")] test_samples = [("test_0.wav", "happy"), ("test_1.wav", "sad")] split_metadata = MediumSplitMetadata( @@ -319,12 +353,13 @@ def __init__( captured["backend_device"] = device captured["backend_dtype"] = dtype + _stub_training_readiness(monkeypatch) monkeypatch.setattr(profile_runtime, "WhisperBackend", _BackendStub) monkeypatch.setattr(em, "reload_settings", lambda: settings) monkeypatch.setattr( training_entrypoints._data_loader, "load_utterances", - lambda *, settings=None: all_utterances, + lambda *, settings=None, allow_prepare=False: all_utterances, ) monkeypatch.setattr( training_support, @@ -418,11 +453,14 @@ def test_resolve_accurate_research_model_id_uses_configured_value( """Accurate-research model id resolver should honor configured settings value.""" del monkeypatch settings = _settings_stub(tmp_path) - settings.models.accurate_research_model_id = "unit-test/emotion2vec-plus" - assert ( - resolve_accurate_research_model_id(cast(AppConfig, settings)) - == "unit-test/emotion2vec-plus" + settings = replace( + settings, + models=replace( + settings.models, + accurate_research_model_id="unit-test/emotion2vec-plus", + ), ) + assert resolve_accurate_research_model_id(settings) == "unit-test/emotion2vec-plus" def test_train_accurate_research_model_persists_emotion2vec_profile_metadata( @@ -431,7 +469,13 @@ def test_train_accurate_research_model_persists_emotion2vec_profile_metadata( ) -> None: """Accurate-research training should persist emotion2vec/accurate-research metadata.""" settings = _settings_stub(tmp_path) - settings.models.accurate_research_model_id = "unit-test/emotion2vec-plus" + settings = replace( + settings, + models=replace( + settings.models, + accurate_research_model_id="unit-test/emotion2vec-plus", + ), + ) train_samples = [("train_0.wav", "happy"), ("train_1.wav", "sad")] test_samples = [("test_0.wav", "happy"), ("test_1.wav", "sad")] split_metadata = MediumSplitMetadata( @@ -465,12 +509,13 @@ def __init__( captured["backend_modelscope_cache_root"] = modelscope_cache_root captured["backend_huggingface_cache_root"] = huggingface_cache_root + _stub_training_readiness(monkeypatch) monkeypatch.setattr(profile_runtime, "Emotion2VecBackend", _BackendStub) monkeypatch.setattr(em, "reload_settings", lambda: settings) monkeypatch.setattr( training_entrypoints._data_loader, "load_utterances", - lambda *, settings=None: all_utterances, + lambda *, settings=None, allow_prepare=False: all_utterances, ) monkeypatch.setattr( training_support, diff --git a/tests/suites/unit/models/test_accurate_training_preparation.py b/tests/suites/unit/models/test_accurate_training_preparation.py index bec2c5f..02adfd2 100644 --- a/tests/suites/unit/models/test_accurate_training_preparation.py +++ b/tests/suites/unit/models/test_accurate_training_preparation.py @@ -455,6 +455,7 @@ def test_run_accurate_profile_training_from_prepared_delegates_to_orchestration( runtime_dtype="float32", x_train=np.asarray([[0.1, 0.2]], dtype=np.float64), y_train=["happy"], + train_meta=[{"sample_id": utterances[0].sample_id}], x_test=np.asarray([[0.3, 0.4]], dtype=np.float64), y_test=["sad"], test_meta=[{"sample_id": utterances[1].sample_id}], diff --git a/tests/suites/unit/models/test_artifact_persistence.py b/tests/suites/unit/models/test_artifact_persistence.py index 1d37f6f..07d0b61 100644 --- a/tests/suites/unit/models/test_artifact_persistence.py +++ b/tests/suites/unit/models/test_artifact_persistence.py @@ -2,12 +2,29 @@ from __future__ import annotations +import errno from dataclasses import dataclass from pathlib import Path +from types import SimpleNamespace +from typing import cast +import numpy as np +import pytest from sklearn.neural_network import MLPClassifier -from ser._internal.models import artifact_persistence +from ser._internal.models import artifact_persistence # noqa: TID251 +from ser._internal.models.training_execution import ( # noqa: TID251 + finalize_profile_training_report, +) +from ser._internal.models.training_orchestration import training_operation_scope # noqa: TID251 +from ser._internal.models.training_readiness import ( # noqa: TID251 + OptionalArtifactError, + TrainingOperation, +) +from ser._internal.models.training_types import ( # noqa: TID251 + PersistedArtifactsLike, + TrainingEvaluation, +) @dataclass(frozen=True) @@ -57,3 +74,166 @@ def _persist_secure(path: Path, model: object) -> bool: Path("models/ser_model.pkl"), Path("models/ser_model.skops"), ) + + +def test_secure_artifact_continues_only_for_explicit_optional_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + model = MLPClassifier(hidden_layer_sizes=(1,), max_iter=1, random_state=0) + + def _raise_optional(_model: object, _path: str) -> None: + raise OptionalArtifactError("advisory format unavailable") + + monkeypatch.setattr( + artifact_persistence.importlib, + "import_module", + lambda _name: SimpleNamespace(dump=_raise_optional), + ) + with training_operation_scope(TrainingOperation()) as state: + assert ( + artifact_persistence.persist_secure_artifact(tmp_path / "model.skops", model) is False + ) + assert state.containment_counts == { + "optional_artifact:optional_artifact_failed:continue": 1 + } + + def _raise_unknown(_model: object, _path: str) -> None: + raise RuntimeError("serializer invariant failed") + + monkeypatch.setattr( + artifact_persistence.importlib, + "import_module", + lambda _name: SimpleNamespace(dump=_raise_unknown), + ) + with pytest.raises(RuntimeError, match="serializer invariant"): + artifact_persistence.persist_secure_artifact(tmp_path / "model.skops", model) + + +def test_secure_artifact_contains_only_target_local_write_errors( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "model.skops" + model = MLPClassifier(hidden_layer_sizes=(1,), max_iter=1, random_state=0) + + def _raise_target_write(_model: object, path: str) -> None: + raise OSError(errno.ENOSPC, "disk full", path) + + monkeypatch.setattr( + artifact_persistence.importlib, + "import_module", + lambda _name: SimpleNamespace(dump=_raise_target_write), + ) + with training_operation_scope(TrainingOperation()) as state: + assert artifact_persistence.persist_secure_artifact(target, model) is False + assert state.containment_counts == { + "optional_artifact:optional_artifact_failed:continue": 1 + } + + def _raise_other_path(_model: object, _path: str) -> None: + raise OSError(errno.ENOSPC, "disk full", str(tmp_path / "required.pkl")) + + monkeypatch.setattr( + artifact_persistence.importlib, + "import_module", + lambda _name: SimpleNamespace(dump=_raise_other_path), + ) + with pytest.raises(OSError) as error: + artifact_persistence.persist_secure_artifact(target, model) + assert error.value.filename == str(tmp_path / "required.pkl") + + +def test_secure_artifact_serialization_value_error_remains_aborting( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def _raise_value_error(_model: object, _path: str) -> None: + raise ValueError("unsupported estimator graph") + + monkeypatch.setattr( + artifact_persistence.importlib, + "import_module", + lambda _name: SimpleNamespace(dump=_raise_value_error), + ) + with pytest.raises(ValueError, match="unsupported estimator graph"): + artifact_persistence.persist_secure_artifact( + tmp_path / "model.skops", + MLPClassifier(), + ) + + def _raise_transitive_missing(_name: str) -> object: + raise ModuleNotFoundError("No module named 'required_runtime'", name="required_runtime") + + monkeypatch.setattr( + artifact_persistence.importlib, + "import_module", + _raise_transitive_missing, + ) + with pytest.raises(ModuleNotFoundError, match="required_runtime"): + artifact_persistence.persist_secure_artifact( + tmp_path / "model.skops", + MLPClassifier(), + ) + + +@pytest.mark.parametrize("failure", ["missing_skops", "target_write"]) +def test_optional_secure_failure_is_durable_in_final_training_report( + failure: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + if failure == "missing_skops": + + def _missing(_name: str) -> object: + raise ModuleNotFoundError("No module named 'skops'", name="skops") + + monkeypatch.setattr(artifact_persistence.importlib, "import_module", _missing) + else: + + def _raise_target_write(_model: object, path: str) -> None: + raise OSError(errno.ENOSPC, "disk full", path) + + monkeypatch.setattr( + artifact_persistence.importlib, + "import_module", + lambda _name: SimpleNamespace(dump=_raise_target_write), + ) + + persisted = SimpleNamespace( + pickle_path=tmp_path / "model.pkl", + secure_path=None, + ) + with training_operation_scope(TrainingOperation()): + assert ( + artifact_persistence.persist_secure_artifact( + tmp_path / "model.skops", + MLPClassifier(), + ) + is False + ) + report = finalize_profile_training_report( + profile_label="Fixture", + logger=artifact_persistence.logger, + evaluation=TrainingEvaluation(1.0, 1.0, 1.0, {}), + ser_metrics={}, + artifact_metadata={}, + persisted_artifacts=cast(PersistedArtifactsLike, persisted), + x_train=np.ones((2, 2), dtype=np.float64), + x_test=np.ones((1, 2), dtype=np.float64), + y_train=["calm", "happy"], + y_test=["calm"], + provenance={"training_robustness": {"statistics": {"containment": {}}}}, + data_controls={}, + build_training_report=lambda **kwargs: kwargs, + persist_training_report=lambda _report: None, + report_destination=tmp_path / "report.json", + ) + + provenance = report["provenance"] + assert isinstance(provenance, dict) + robustness = provenance["training_robustness"] + assert isinstance(robustness, dict) + statistics = robustness["statistics"] + assert isinstance(statistics, dict) + assert statistics["containment"] == {"optional_artifact:optional_artifact_failed:continue": 1} diff --git a/tests/suites/unit/models/test_medium_training_preparation.py b/tests/suites/unit/models/test_medium_training_preparation.py index 8ebace6..8340d5e 100644 --- a/tests/suites/unit/models/test_medium_training_preparation.py +++ b/tests/suites/unit/models/test_medium_training_preparation.py @@ -152,6 +152,7 @@ def test_run_medium_profile_training_from_prepared_delegates_to_orchestration( runtime_dtype="float32", x_train=np.asarray([[0.1, 0.2]], dtype=np.float64), y_train=["happy"], + train_meta=[{"sample_id": utterances[0].sample_id}], x_test=np.asarray([[0.3, 0.4]], dtype=np.float64), y_test=["sad"], test_meta=[{"sample_id": utterances[1].sample_id}], diff --git a/tests/suites/unit/models/test_training_mode_orchestration.py b/tests/suites/unit/models/test_training_mode_orchestration.py new file mode 100644 index 0000000..e2a6818 --- /dev/null +++ b/tests/suites/unit/models/test_training_mode_orchestration.py @@ -0,0 +1,1036 @@ +"""Train-mode CLI and mandatory-check orchestration regressions.""" + +from __future__ import annotations + +import errno +import logging +import threading +import time +from collections.abc import Sequence +from dataclasses import replace +from pathlib import Path +from typing import cast + +import numpy as np +import pytest +from sklearn.neural_network import MLPClassifier + +import ser._internal.data.data_loader as data_loader # noqa: TID251 +import ser._internal.diagnostics.service as diagnostics_service # noqa: TID251 +import ser._internal.models.dataset_splitting as dataset_splitting # noqa: TID251 +import ser._internal.models.fast_training as fast_training # noqa: TID251 +import ser._internal.models.profile_training_preparation as profile_preparation # noqa: TID251 +import ser._internal.models.training_entrypoints as training_entrypoints # noqa: TID251 +import ser._internal.models.training_orchestration as orchestration # noqa: TID251 +from ser.__main__ import ( + _build_main_parser, + _training_operation_from_args, + _validate_training_mode_args, +) +from ser._internal.config.schema import ( # noqa: TID251 + DataLoaderConfig, + DatasetConfig, + ModelsConfig, + TrainingConfig, +) +from ser._internal.data.dataset_registry import DatasetRegistryEntry # noqa: TID251 +from ser._internal.data.manifest import Utterance # noqa: TID251 +from ser._internal.models.medium_noise_controls import ( # noqa: TID251 + MediumNoiseControlStats, + merge_medium_noise_stats, +) +from ser._internal.models.training_orchestration import ( # noqa: TID251 + training_operation_scope, +) +from ser._internal.models.training_readiness import ( # noqa: TID251 + PreparedPlanError, + QuarantineBudgetExceeded, + ReadinessReport, + TrainingMode, + TrainingOperation, + TrainingReadinessError, +) +from ser._internal.models.training_support import WindowMeta # noqa: TID251 +from ser._internal.repr import FeatureBackend # noqa: TID251 +from ser._internal.repr import EncodedSequence, PoolingWindow # noqa: TID251 +from ser._internal.runtime.commands import classify_training_exception # noqa: TID251 +from ser._internal.utils.audio_utils import AudioDecodeError # noqa: TID251 +from ser.config import AppConfig +from ser.diagnostics.domain import DiagnosticFinding + + +def _settings() -> AppConfig: + return AppConfig(emotions={"01": "calm", "02": "happy"}) + + +def test_cli_parses_each_training_mode_and_prepared_plan(tmp_path: Path) -> None: + parser = _build_main_parser(_settings()) + dry_args = parser.parse_args(["--train", "--dry-run"]) + prepare_args = parser.parse_args(["--train", "--prepare-only", "--repair"]) + plan_args = parser.parse_args(["--train", "--prepared-plan", str(tmp_path / "plan.json")]) + _validate_training_mode_args(parser, dry_args) + _validate_training_mode_args(parser, prepare_args) + _validate_training_mode_args(parser, plan_args) + assert _training_operation_from_args(dry_args).mode is TrainingMode.DRY_RUN + assert _training_operation_from_args(prepare_args) == TrainingOperation( + mode=TrainingMode.PREPARE_ONLY, + repair=True, + ) + assert _training_operation_from_args(plan_args).prepared_plan == tmp_path / "plan.json" + + +@pytest.mark.parametrize( + "argv", + [ + ["--dry-run"], + ["--train", "--repair"], + ["--train", "--dry-run", "--prepared-plan", "plan.json"], + ], +) +def test_cli_rejects_train_mode_conflicts_before_execution(argv: list[str]) -> None: + parser = _build_main_parser(_settings()) + args = parser.parse_args(argv) + with pytest.raises(SystemExit) as raised: + _validate_training_mode_args(parser, args) + assert raised.value.code == 2 + + +def test_argparse_rejects_dry_run_and_prepare_only_together() -> None: + parser = _build_main_parser(_settings()) + with pytest.raises(SystemExit) as raised: + parser.parse_args(["--train", "--dry-run", "--prepare-only"]) + assert raised.value.code == 2 + + +def test_dry_run_returns_after_shared_checker_before_profile_training( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + monkeypatch.setattr( + training_entrypoints._training_orchestration, + "ensure_entrypoint_readiness", + lambda **_kwargs: calls.append("checked"), + ) + monkeypatch.setattr( + training_entrypoints._fast_training_entrypoints, + "train_model", + lambda **_kwargs: calls.append("trained"), + ) + with training_operation_scope(TrainingOperation(mode=TrainingMode.DRY_RUN)): + training_entrypoints.train_model(settings=_settings()) + assert calls == ["checked"] + + +def test_real_training_invokes_shared_checker_before_profile_entrypoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + monkeypatch.setattr( + training_entrypoints._training_orchestration, + "ensure_entrypoint_readiness", + lambda **_kwargs: calls.append("checked"), + ) + monkeypatch.setattr( + training_entrypoints._fast_training_entrypoints, + "train_model", + lambda **_kwargs: calls.append("trained"), + ) + with training_operation_scope(TrainingOperation()): + training_entrypoints.train_model(settings=_settings()) + assert calls == ["checked", "trained"] + + +def test_readiness_loader_never_rebuilds_missing_manifest_without_repair( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _settings() + entry = DatasetRegistryEntry( + dataset_id="fixture", + dataset_root=tmp_path / "dataset", + manifest_path=tmp_path / "missing.jsonl", + options={}, + ) + monkeypatch.setattr(data_loader, "load_dataset_registry", lambda **_kwargs: {"fixture": entry}) + monkeypatch.setattr(data_loader, "validate_registered_dataset_integrity", lambda _entry: None) + prepare_called = False + + def _prepare(**_kwargs: object) -> tuple[Path, ...]: + nonlocal prepare_called + prepare_called = True + return () + + monkeypatch.setattr(data_loader, "prepare_from_registry_entry", _prepare) + with pytest.raises(RuntimeError, match="missing its manifest"): + data_loader.load_utterances(settings=settings, allow_prepare=False) + assert prepare_called is False + + +def test_backend_smoke_has_a_hard_wall_clock_deadline( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("SER_TRAINING_SMOKE_TIMEOUT_SECONDS", "0.01") + + def _slow_smoke(**_kwargs: object) -> object: + time.sleep(1.0) + raise AssertionError("deadline failed") + + monkeypatch.setattr(orchestration, "_run_selected_backend_smoke_unbounded", _slow_smoke) + started = time.perf_counter() + with pytest.raises(TimeoutError, match="hard wall-clock timeout"): + orchestration.run_selected_backend_smoke( + settings=_settings(), + samples=(), + probe_cache_dir=tmp_path, + ) + assert time.perf_counter() - started < 0.5 + + +def test_backend_smoke_rejects_worker_without_hard_deadline(tmp_path: Path) -> None: + failures: list[BaseException] = [] + + def _run() -> None: + try: + orchestration.run_selected_backend_smoke( + settings=_settings(), + samples=(), + probe_cache_dir=tmp_path, + ) + except BaseException as error: + failures.append(error) + + worker = threading.Thread(target=_run) + worker.start() + worker.join(timeout=1.0) + assert not worker.is_alive() + assert len(failures) == 1 + assert isinstance(failures[0], RuntimeError) + assert "hard-deadline" in str(failures[0]) + + +def test_model_revision_must_come_from_checked_backend() -> None: + class _ResolvedBackend: + backend_id = "hf_xlsr" + feature_dim = 3 + model_revision = "commit-a" + + with training_operation_scope(TrainingOperation()) as state: + state.checked_backend = cast(FeatureBackend, _ResolvedBackend()) + state.checked_backend_id = "hf_xlsr" + state.checked_model_id = "repo/model" + assert ( + orchestration._resolved_model_revision(backend_id="hf_xlsr", model_id="repo/model") + == "commit-a" + ) + _ResolvedBackend.model_revision = "repo/model" + with pytest.raises(PreparedPlanError, match="unpinned"): + orchestration._resolved_model_revision(backend_id="hf_xlsr", model_id="repo/model") + + +def test_transient_local_io_retry_is_bounded_and_accounted( + monkeypatch: pytest.MonkeyPatch, +) -> None: + attempts = 0 + monkeypatch.setattr(orchestration.time, "sleep", lambda _seconds: None) + + def _operation() -> str: + nonlocal attempts + attempts += 1 + if attempts < 3: + raise InterruptedError("transient read") + return "ok" + + with training_operation_scope(TrainingOperation()) as state: + assert orchestration.bounded_retry_local_io(_operation, identity="sample") == "ok" + assert state.bounded_retries == 2 + attempts = 0 + with training_operation_scope(TrainingOperation()): + with pytest.raises(InterruptedError): + orchestration.bounded_retry_local_io( + _operation, + identity="sample", + max_retries=1, + ) + + +def test_transient_errno_is_retried_but_resource_exhaustion_aborts( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(orchestration.time, "sleep", lambda _seconds: None) + attempts = 0 + + def _transient() -> str: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise OSError(errno.EAGAIN, "temporarily unavailable") + return "ok" + + with training_operation_scope(TrainingOperation()) as state: + assert orchestration.bounded_retry_local_io(_transient, identity="sample") == "ok" + assert state.bounded_retries == 1 + assert state.containment_counts == {"sample:media_decode_failed:bounded_retry": 1} + with pytest.raises(OSError) as error: + orchestration.bounded_retry_local_io( + lambda: (_ for _ in ()).throw(OSError(errno.ENOSPC, "disk full")), + identity="sample", + ) + assert error.value.errno == errno.ENOSPC + + +def test_encoding_quarantine_rechecks_canonical_dev_support(tmp_path: Path) -> None: + settings = AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + training=TrainingConfig(test_size=0.25, dev_size=0.25), + data_loader=DataLoaderConfig( + max_failed_files=12, + max_failed_file_ratio=1.0, + max_failed_file_ratio_per_corpus=1.0, + max_failed_file_ratio_per_class=1.0, + max_failures_per_reason=12, + min_remaining_per_class_split=1, + ), + ) + samples = tuple( + Utterance( + 1, + f"sample-{index}", + "fixture", + tmp_path / f"{index}.wav", + "calm" if index % 2 == 0 else "happy", + ) + for index in range(12) + ) + train, _, _, _ = dataset_splitting.split_utterances_three_way( + samples=list(samples), + settings=settings, + logger=orchestration.logger, + ) + train_support = { + label: [item for item in train if item.label == label] for label in ("calm", "happy") + } + safe_candidate = next(items[0] for items in train_support.values() if len(items) > 1) + + def _encode(item: Utterance) -> EncodedSequence: + if item.sample_id == safe_candidate.sample_id: + raise AudioDecodeError("isolated decoder failure") + return cast(EncodedSequence, object()) + + def _windows( + _encoded: EncodedSequence, + _window_size: float, + _window_stride: float, + ) -> list[PoolingWindow]: + return [] + + def _pool( + _encoded: EncodedSequence, + _windows: Sequence[PoolingWindow], + ) -> np.ndarray: + return np.asarray([[0.1, 0.2]], dtype=np.float64) + + def _apply(features: np.ndarray) -> tuple[np.ndarray, MediumNoiseControlStats]: + return features, MediumNoiseControlStats(total_windows=1, kept_windows=1) + + def _prepare() -> tuple[np.ndarray, list[str], list[WindowMeta], MediumNoiseControlStats]: + return profile_preparation.build_medium_feature_dataset( + utterances=orchestration.current_training_state().utterances, + encode_sequence=_encode, + window_size_seconds=1.0, + window_stride_seconds=1.0, + build_pooling_windows=_windows, + pool_features=_pool, + apply_noise_controls=_apply, + merge_noise_stats=merge_medium_noise_stats, + initial_noise_stats=MediumNoiseControlStats(), + window_meta_factory=WindowMeta, + handle_sample_failure=lambda item, error: orchestration.handle_sample_encoding_failure( + settings=settings, + sample=item, + error=error, + ), + ) + + with training_operation_scope(TrainingOperation()) as state: + state.utterances = samples + matrix, labels, metadata, stats = orchestration.prepare_until_quarantine_stable( + settings=settings, + prepare=_prepare, + ) + assert safe_candidate.sample_id not in {item.sample_id for item in state.utterances} + assert matrix.shape[0] == len(samples) - 1 + assert len(labels) == len(metadata) == len(samples) - 1 + assert stats.kept_windows == len(samples) - 1 + fit_calls = 0 + + class _FitProbe: + def fit(self, rows: np.ndarray, targets: list[str]) -> None: + nonlocal fit_calls + assert rows.shape[0] == len(targets) == len(samples) - 1 + fit_calls += 1 + + _FitProbe().fit(matrix, labels) + assert fit_calls == 1 + + with training_operation_scope(TrainingOperation()) as state: + minimal_samples = samples[:6] + state.utterances = minimal_samples + failing_sample = minimal_samples[0] + blocked_fit_calls = 0 + + def _fail_minimal(item: Utterance) -> EncodedSequence: + if item.sample_id == failing_sample.sample_id: + raise AudioDecodeError("isolated decoder failure") + return cast(EncodedSequence, object()) + + with pytest.raises(QuarantineBudgetExceeded, match="Projected (dev split|canonical split)"): + profile_preparation.build_medium_feature_dataset( + utterances=state.utterances, + encode_sequence=_fail_minimal, + window_size_seconds=1.0, + window_stride_seconds=1.0, + build_pooling_windows=_windows, + pool_features=_pool, + apply_noise_controls=_apply, + merge_noise_stats=merge_medium_noise_stats, + initial_noise_stats=MediumNoiseControlStats(), + window_meta_factory=WindowMeta, + handle_sample_failure=lambda item, error: ( + orchestration.handle_sample_encoding_failure( + settings=settings, + sample=item, + error=error, + ) + ), + ) + assert blocked_fit_calls == 0 + + with training_operation_scope(TrainingOperation()) as state: + state.utterances = samples + + def _systematic_failure(_item: Utterance) -> EncodedSequence: + raise RuntimeError("backend contract failure") + + with pytest.raises(RuntimeError, match="backend contract failure"): + profile_preparation.build_medium_feature_dataset( + utterances=state.utterances, + encode_sequence=_systematic_failure, + window_size_seconds=1.0, + window_stride_seconds=1.0, + build_pooling_windows=_windows, + pool_features=_pool, + apply_noise_controls=_apply, + merge_noise_stats=merge_medium_noise_stats, + initial_noise_stats=MediumNoiseControlStats(), + window_meta_factory=WindowMeta, + handle_sample_failure=lambda item, error: ( + orchestration.handle_sample_encoding_failure( + settings=settings, + sample=item, + error=error, + ) + ), + ) + assert blocked_fit_calls == 0 + + +def test_checked_backend_is_reused_once_and_closed_at_scope_exit() -> None: + closed = 0 + builds = 0 + + class _Backend: + backend_id = "fixture" + feature_dim = 3 + + def close(self) -> None: + nonlocal closed + closed += 1 + + backend = cast(FeatureBackend, _Backend()) + + def _build() -> FeatureBackend: + nonlocal builds + builds += 1 + return backend + + with training_operation_scope(TrainingOperation()): + first = orchestration.reuse_checked_backend( + backend_id="fixture", + model_id="model@revision", + device="cpu", + dtype="float32", + build=_build, + ) + second = orchestration.reuse_checked_backend( + backend_id="fixture", + model_id="model@revision", + device="cpu", + dtype="float32", + build=_build, + ) + assert first is second + assert builds == 1 + assert closed == 1 + + +def test_fast_preparation_consumes_exactly_the_checked_utterance_view( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _settings() + checked = [ + Utterance(1, "calm-1", "fixture", tmp_path / "one.wav", "calm"), + Utterance(1, "calm-2", "fixture", tmp_path / "two.wav", "calm"), + Utterance(1, "happy-1", "fixture", tmp_path / "three.wav", "happy"), + Utterance(1, "happy-2", "fixture", tmp_path / "four.wav", "happy"), + ] + reads: list[str] = [] + + def _read(path: str, **_kwargs: object) -> tuple[np.ndarray, int]: + reads.append(path) + return np.ones(80, dtype=np.float32), 8000 + + class _Backend: + def __init__(self, **_kwargs: object) -> None: + pass + + def extract_vector(self, *, audio: np.ndarray, sample_rate: int) -> np.ndarray: + assert sample_rate == 8000 + return np.asarray([float(audio.size)], dtype=np.float64) + + monkeypatch.setattr(data_loader, "read_audio_file", _read) + monkeypatch.setattr(data_loader, "HandcraftedBackend", _Backend) + split = data_loader.load_checked_fast_data(utterances=checked, settings=settings) + assert split is not None + x_train, x_test, y_train, y_test = split + assert x_train.shape[0] + x_test.shape[0] == len(checked) + assert sorted(reads) == sorted(str(item.audio_path) for item in checked) + assert sorted([*y_train, *y_test]) == ["calm", "calm", "happy", "happy"] + + +def test_operation_level_plan_validation_recomputes_trusted_expectations( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + training=TrainingConfig(test_size=0.25, dev_size=0.25), + models=ModelsConfig(folder=tmp_path / "models", model_cache_dir=tmp_path / "cache"), + ) + utterances = tuple( + Utterance( + 1, + f"sample-{index}", + "fixture", + tmp_path / f"{index}.wav", + "calm" if index % 2 == 0 else "happy", + ) + for index in range(8) + ) + readiness = ReadinessReport( + 1, + "2026-01-01T00:00:00+00:00", + "fast", + "settings", + "registry", + "manifest", + "media", + (), + (), + effective_sample_ids=tuple(item.sample_id for item in utterances), + ) + with training_operation_scope(TrainingOperation(mode=TrainingMode.PREPARE_ONLY)) as state: + state.readiness = readiness + state.utterances = utterances + orchestration.publish_prepared_features( + settings=settings, + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + utterances=utterances, + x_train=np.ones((6, 3), dtype=np.float64), + x_test=np.ones((2, 3), dtype=np.float64), + y_train=["calm", "happy", "calm", "happy", "calm", "happy"], + y_test=["calm", "happy"], + metadata={"split_metadata": {}}, + cache_namespace="fast_features", + windowing_policy={"strategy": "utterance"}, + noise_statistics={}, + ) + plan_path = settings.tmp_folder / "prepared-training-fast.json" + plan = orchestration.load_prepared_plan(plan_path) + assert plan.effective_counts["partition_samples"] == {"train": 4, "test": 2, "dev": 2} + assert plan.disposition_counts == { + "included_samples": 8, + "quarantined_samples": 0, + "included_windows": 8, + "dropped_windows": 0, + } + for namespace in ("../escape", str(tmp_path / "absolute")): + forged = replace(plan, cache_namespace=namespace) + monkeypatch.setattr( + orchestration, + "prepared_plan_for_operation", + lambda _settings, candidate=forged: candidate, + ) + with training_operation_scope(TrainingOperation()) as state: + state.readiness = readiness + state.utterances = utterances + with pytest.raises(PreparedPlanError, match="namespace"): + orchestration.validate_operation_plan( + settings=settings, + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + ) + forged_version = replace(plan, cache_version="attacker-controlled") + monkeypatch.setattr( + orchestration, + "prepared_plan_for_operation", + lambda _settings: forged_version, + ) + with training_operation_scope(TrainingOperation()) as state: + state.readiness = readiness + state.utterances = utterances + with pytest.raises(PreparedPlanError, match="cache_version"): + orchestration.validate_operation_plan( + settings=settings, + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + ) + + monkeypatch.setattr( + orchestration, + "prepared_plan_for_operation", + lambda _settings: orchestration.load_prepared_plan(plan_path), + ) + added_cache = settings.tmp_folder / "fast_features" / "unexpected.npz" + added_cache.parent.mkdir(parents=True) + added_cache.write_bytes(b"new cache key") + with training_operation_scope(TrainingOperation(prepared_plan=plan_path)) as state: + state.readiness = readiness + state.utterances = utterances + with pytest.raises(PreparedPlanError, match="cache_keys"): + orchestration.validate_operation_plan( + settings=settings, + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + ) + + +def test_fast_prepare_only_publishes_three_way_plan_without_classifier( + tmp_path: Path, +) -> None: + settings = AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + training=TrainingConfig(test_size=0.25, dev_size=0.25), + models=ModelsConfig(folder=tmp_path / "models", model_cache_dir=tmp_path / "cache"), + ) + utterances = tuple( + Utterance( + 1, + f"sample-{index}", + "fixture", + tmp_path / f"{index}.wav", + "calm" if index % 2 == 0 else "happy", + ) + for index in range(8) + ) + readiness = ReadinessReport( + 1, + "2026-01-01T00:00:00+00:00", + "fast", + "settings", + "registry", + "manifest", + "media", + (), + (), + effective_sample_ids=tuple(item.sample_id for item in utterances), + ) + initial_train, initial_test, _ = dataset_splitting.split_utterances( + samples=list(utterances), settings=settings, logger=orchestration.logger + ) + split = ( + np.arange(len(initial_train) * 3, dtype=np.float64).reshape(-1, 3), + np.arange(len(initial_test) * 3, dtype=np.float64).reshape(-1, 3), + [item.require_label() for item in initial_train], + [item.require_label() for item in initial_test], + ) + classifier_calls = 0 + + def _create_classifier() -> MLPClassifier: + nonlocal classifier_calls + classifier_calls += 1 + return MLPClassifier() + + def _unexpected_evaluation( + *, y_true: list[str], y_pred: list[str] + ) -> fast_training.TrainingEvaluationLike: + raise AssertionError("prepare-only evaluated a classifier") + + def _unexpected_persistence( + *, model: fast_training.EmotionClassifier, artifact: dict[str, object] + ) -> fast_training.PersistedArtifactsLike: + raise AssertionError("prepare-only persisted a model") + + def _ignore_training_report(report: dict[str, object], path: Path) -> None: + del report, path + + hooks = fast_training.FastTrainingHooks( + logger=orchestration.logger, + settings=settings, + load_utterances=lambda: utterances, + ensure_dataset_consents_for_training=lambda *, utterances: None, + load_data=lambda *, test_size: split, + load_checked_data=lambda _utterances: split, + create_classifier=_create_classifier, + evaluate_training_predictions=_unexpected_evaluation, + build_provenance_metadata=lambda **_kwargs: {}, + build_model_artifact=lambda **_kwargs: {}, + extract_artifact_metadata=lambda _artifact: {}, + persist_model_artifacts=_unexpected_persistence, + build_training_report=lambda **_kwargs: {}, + persist_training_report=_ignore_training_report, + default_backend_id="handcrafted", + default_profile_id="fast", + ) + with training_operation_scope(TrainingOperation(mode=TrainingMode.PREPARE_ONLY)) as state: + state.readiness = readiness + state.utterances = utterances + fast_training.train_fast_model(hooks=hooks) + + plan = orchestration.load_prepared_plan(settings.tmp_folder / "prepared-training-fast.json") + with training_operation_scope( + TrainingOperation(prepared_plan=Path(plan.payload_path)) + ) as state: + state.readiness = readiness + state.utterances = utterances + payload = orchestration.read_prepared_feature_payload(plan) + assert classifier_calls == 0 + assert (payload.x_train.shape[0], payload.x_dev.shape[0], payload.x_test.shape[0]) == ( + 4, + 2, + 2, + ) + + class _Evaluation: + accuracy = 1.0 + macro_f1 = 1.0 + uar = 1.0 + ser_metrics: dict[str, object] = {} + + class _Persisted: + pickle_path = tmp_path / "model.pkl" + secure_path: Path | None = None + + captures: list[tuple[np.ndarray, list[str]]] = [] + + class _Classifier: + def fit(self, x_rows: np.ndarray, labels: list[str]) -> None: + captures.append((np.asarray(x_rows), list(labels))) + + def predict(self, x_rows: np.ndarray) -> np.ndarray: + return np.asarray(["calm"] * len(x_rows)) + + def _training_hooks() -> fast_training.FastTrainingHooks: + return replace( + hooks, + create_classifier=lambda: cast( + fast_training.EmotionClassifier, + _Classifier(), + ), + evaluate_training_predictions=lambda *, y_true, y_pred: _Evaluation(), + build_provenance_metadata=lambda **_kwargs: {}, + build_model_artifact=lambda **_kwargs: {}, + extract_artifact_metadata=lambda _artifact: {}, + persist_model_artifacts=lambda **_kwargs: _Persisted(), + build_training_report=lambda **_kwargs: {}, + ) + + with training_operation_scope(TrainingOperation()) as state: + state.readiness = readiness + state.utterances = utterances + fast_training.train_fast_model(hooks=_training_hooks()) + plan_path = settings.tmp_folder / "prepared-training-fast.json" + with training_operation_scope(TrainingOperation(prepared_plan=plan_path)) as state: + state.readiness = readiness + state.utterances = utterances + fast_training.train_fast_model(hooks=_training_hooks()) + + assert len(captures) == 2 + np.testing.assert_array_equal(captures[0][0], captures[1][0]) + assert captures[0][1] == captures[1][1] + + +def test_prepared_plan_matches_full_fresh_partition_and_noise_provenance( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + settings = AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + training=TrainingConfig(test_size=0.25, dev_size=0.25), + models=ModelsConfig(folder=tmp_path / "models", model_cache_dir=tmp_path / "cache"), + ) + utterances = tuple( + Utterance( + 1, + f"sample-{index}", + "fixture", + tmp_path / f"{index}.wav", + "calm" if index % 2 == 0 else "happy", + ) + for index in range(8) + ) + readiness = ReadinessReport( + 1, + "2026-01-01T00:00:00+00:00", + "fast", + "settings", + "registry", + "manifest", + "media", + (), + (), + effective_sample_ids=tuple(item.sample_id for item in utterances), + ) + initial_train, initial_test, split_metadata = dataset_splitting.split_utterances( + samples=list(utterances), settings=settings, logger=orchestration.logger + ) + x_train = np.arange(len(initial_train) * 3, dtype=np.float64).reshape(-1, 3) + x_test = np.arange(len(initial_test) * 3, dtype=np.float64).reshape(-1, 3) + y_train = [item.require_label() for item in initial_train] + y_test = [item.require_label() for item in initial_test] + train_meta = [ + {"sample_id": item.sample_id, "corpus": item.corpus, "language": "en"} + for item in initial_train + ] + with training_operation_scope(TrainingOperation()) as state: + state.utterances = utterances + fresh_x, fresh_y, fresh_meta, fresh_utterances = orchestration.canonical_train_partition( + settings=settings, + x_train=x_train, + y_train=y_train, + train_metadata=train_meta, + sample_id=lambda item: item["sample_id"], + ) + fresh_noise = MediumNoiseControlStats( + total_windows=len(fresh_meta) + 2, + kept_windows=len(fresh_meta), + dropped_low_std_windows=1, + dropped_cap_windows=1, + ) + test_noise = MediumNoiseControlStats( + total_windows=len(initial_test), + kept_windows=len(initial_test), + ) + caplog.set_level(logging.INFO, logger=orchestration.logger.name) + with training_operation_scope(TrainingOperation(mode=TrainingMode.PREPARE_ONLY)) as state: + state.readiness = readiness + state.utterances = utterances + orchestration.publish_prepared_features( + settings=settings, + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + utterances=utterances, + x_train=x_train, + x_test=x_test, + y_train=y_train, + y_test=y_test, + metadata={ + "split_metadata": split_metadata, + "train_meta": train_meta, + "train_noise_stats": fresh_noise, + "test_noise_stats": test_noise, + }, + cache_namespace="fast_features", + windowing_policy={"strategy": "fixture"}, + noise_statistics={"train": fresh_noise, "test": test_noise}, + ) + loaded_plan = orchestration.load_prepared_plan( + settings.tmp_folder / "prepared-training-fast.json" + ) + with training_operation_scope( + TrainingOperation(prepared_plan=settings.tmp_folder / "prepared-training-fast.json") + ) as state: + state.readiness = readiness + state.utterances = utterances + payload = orchestration.read_prepared_feature_payload(loaded_plan) + + np.testing.assert_array_equal(payload.x_train, fresh_x) + assert payload.y_train == fresh_y + assert payload.metadata["train_meta"] == fresh_meta + assert payload.metadata["train_sample_ids"] == [item.sample_id for item in fresh_utterances] + expected_noise = orchestration.serialize_metadata(fresh_noise) + assert payload.metadata["train_noise_stats"] == expected_noise + assert loaded_plan.noise_statistics["train"] == expected_noise + messages = [record.getMessage() for record in caplog.records] + assert any(message.startswith("PREPARED_PUBLISH_START") for message in messages) + assert any(message.startswith("PREPARED_PUBLISH_DONE") for message in messages) + + +def test_prepared_payload_validation_rejects_semantic_shape_tampering(tmp_path: Path) -> None: + settings = AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + training=TrainingConfig(test_size=0.25, dev_size=0.25), + models=ModelsConfig(folder=tmp_path / "models", model_cache_dir=tmp_path / "cache"), + ) + utterances = tuple( + Utterance( + 1, + f"sample-{index}", + "fixture", + tmp_path / f"{index}.wav", + "calm" if index % 2 == 0 else "happy", + ) + for index in range(8) + ) + readiness = ReadinessReport( + 1, + "2026-01-01T00:00:00+00:00", + "fast", + "settings", + "registry", + "manifest", + "media", + (), + (), + effective_sample_ids=tuple(item.sample_id for item in utterances), + ) + with training_operation_scope(TrainingOperation(mode=TrainingMode.PREPARE_ONLY)) as state: + state.readiness = readiness + state.utterances = utterances + plan = orchestration.publish_prepared_features( + settings=settings, + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + utterances=utterances, + x_train=np.ones((6, 3), dtype=np.float64), + x_test=np.ones((2, 3), dtype=np.float64), + y_train=["calm", "happy", "calm", "happy", "calm", "happy"], + y_test=["calm", "happy"], + metadata={"split_metadata": {}}, + cache_namespace="fast_features", + windowing_policy={"strategy": "utterance"}, + noise_statistics={}, + ) + payload_path = Path(plan.payload_path) + with payload_path.open("wb") as handle: + np.savez_compressed( + handle, + x_train=np.ones((2, 4), dtype=np.float64), + x_dev=np.ones((2, 4), dtype=np.float64), + x_test=np.ones((2, 4), dtype=np.float64), + y_train=np.asarray(["calm", "happy"]), + y_dev=np.asarray(["calm", "happy"]), + y_test=np.asarray(["calm", "happy"]), + metadata_json=np.asarray("{}"), + ) + tampered = replace(plan, payload_digest=orchestration.hash_file(payload_path)) + tampered = replace( + tampered, + overall_digest=orchestration.digest_payload(tampered.unsigned_dict()), + ) + with pytest.raises(PreparedPlanError, match="planned shape"): + orchestration.read_prepared_feature_payload(tampered) + + +def test_doctor_command_path_includes_the_complete_training_checker( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + diagnostics_service, + "_check_complete_training_readiness", + lambda **_kwargs: ( + DiagnosticFinding( + code="training_readiness_fixture", + severity="info", + message="checked", + ), + ), + ) + report = diagnostics_service.run_doctor_diagnostics( + settings=_settings(), + include_transcription_checks=False, + include_noise_findings=False, + include_training_readiness=True, + ) + assert any(item.code == "training_readiness_fixture" for item in report.findings) + + +def test_training_exit_codes_distinguish_validation_from_internal_failure() -> None: + validation_report = ReadinessReport( + 1, + "2026-01-01T00:00:00+00:00", + "fast", + "settings", + "registry", + "manifest", + "media", + (), + (), + ) + validation = classify_training_exception(TrainingReadinessError(validation_report)) + internal = classify_training_exception(RuntimeError("unexpected invariant defect")) + assert validation.exit_code == 2 + assert validation.include_traceback is False + assert internal.exit_code == 1 + assert internal.include_traceback is True + + +def test_robustness_statistics_are_exposed_for_artifact_and_report_provenance() -> None: + readiness = ReadinessReport( + 1, + "2026-01-01T00:00:00+00:00", + "fast", + "settings", + "registry", + "manifest", + "media", + (), + (), + ) + with training_operation_scope(TrainingOperation()) as state: + state.readiness = readiness + state.cache_hits = 3 + state.cache_misses = 2 + state.recomputed_cache_entries = 1 + state.dropped_windows = 4 + provenance = orchestration.build_training_robustness_provenance() + assert provenance["readiness"] == { + "schema_version": 1, + "settings_digest": "settings", + "registry_digest": "registry", + "manifest_digest": "manifest", + "media_digest": "media", + } + assert provenance["statistics"] == { + "quarantined_samples": 0, + "dropped_windows": 4, + "cache_hits": 3, + "cache_misses": 2, + "recomputed_cache_entries": 1, + "bounded_retries": 0, + "containment": {}, + } diff --git a/tests/suites/unit/models/test_training_resilience_evidence.py b/tests/suites/unit/models/test_training_resilience_evidence.py new file mode 100644 index 0000000..953a4b5 --- /dev/null +++ b/tests/suites/unit/models/test_training_resilience_evidence.py @@ -0,0 +1,180 @@ +"""Integrated evidence for resumable preparation and deterministic resource cleanup.""" + +from __future__ import annotations + +from pathlib import Path +from typing import cast + +import numpy as np +import pytest + +import ser._internal.models.training_readiness as readiness_module # noqa: TID251 +from ser._internal.config.schema import DatasetConfig, ModelsConfig, TrainingConfig # noqa: TID251 +from ser._internal.data.embedding_cache import ( # noqa: TID251 + EmbeddingCache, + EmbeddingCacheEntry, +) +from ser._internal.data.manifest import Utterance # noqa: TID251 +from ser._internal.models import dataset_splitting # noqa: TID251 +from ser._internal.models import training_orchestration as orchestration # noqa: TID251 +from ser._internal.models.training_orchestration import training_operation_scope # noqa: TID251 +from ser._internal.models.training_readiness import ( # noqa: TID251 + ReadinessReport, + TrainingMode, + TrainingOperation, +) +from ser._internal.repr import EncodedSequence, FeatureBackend # noqa: TID251 +from ser.config import AppConfig + + +def _settings(tmp_path: Path) -> AppConfig: + return AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + dataset=DatasetConfig(folder=tmp_path), + training=TrainingConfig(test_size=0.25, dev_size=0.25), + models=ModelsConfig(folder=tmp_path / "models", model_cache_dir=tmp_path / "cache"), + ) + + +def _inventory(tmp_path: Path) -> tuple[Utterance, ...]: + samples: list[Utterance] = [] + for index in range(8): + path = tmp_path / f"{index}.wav" + path.write_bytes(f"audio-{index}".encode()) + samples.append( + Utterance( + 1, + f"sample-{index}", + "fixture", + path, + "calm" if index % 2 == 0 else "happy", + ) + ) + return tuple(samples) + + +def _readiness(samples: tuple[Utterance, ...]) -> ReadinessReport: + return ReadinessReport( + 1, + "2026-01-01T00:00:00+00:00", + "fast", + "settings", + "registry", + "manifest", + "media", + (), + (), + effective_sample_ids=tuple(item.sample_id for item in samples), + ) + + +def test_interrupted_prepare_has_no_ready_plan_and_rerun_reuses_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = _settings(tmp_path) + samples = _inventory(tmp_path) + readiness = _readiness(samples) + cache = EmbeddingCache(settings.tmp_folder / "fast_features") + compute_calls = 0 + + def _compute() -> EncodedSequence: + nonlocal compute_calls + compute_calls += 1 + return EncodedSequence( + embeddings=np.asarray([[0.1, 0.2]], dtype=np.float32), + frame_start_seconds=np.asarray([0.0], dtype=np.float64), + frame_end_seconds=np.asarray([0.1], dtype=np.float64), + backend_id="handcrafted", + ) + + def _cache_lookup() -> EmbeddingCacheEntry: + return cache.get_or_compute( + audio_path=str(samples[0].audio_path), + backend_id="handcrafted", + model_id="builtin", + frame_size_seconds=1.0, + frame_stride_seconds=1.0, + compute=_compute, + ) + + first = _cache_lookup() + assert first.cache_hit is False + + original_replace = readiness_module.os.replace + interrupted = False + + def _interrupt_plan(source: str | Path, destination: str | Path) -> None: + nonlocal interrupted + if Path(destination).name == "prepared-training-fast.json" and not interrupted: + interrupted = True + raise OSError("simulated plan publication interruption") + original_replace(source, destination) + + monkeypatch.setattr(readiness_module.os, "replace", _interrupt_plan) + initial_train, initial_test, _ = dataset_splitting.split_utterances( + samples=list(samples), settings=settings, logger=orchestration.logger + ) + x_train = np.ones((len(initial_train), 2), dtype=np.float64) + x_test = np.ones((len(initial_test), 2), dtype=np.float64) + + def _publish() -> None: + orchestration.publish_prepared_features( + settings=settings, + backend_id="handcrafted", + model_id="builtin", + device="cpu", + dtype="float64", + utterances=samples, + x_train=x_train, + x_test=x_test, + y_train=[item.require_label() for item in initial_train], + y_test=[item.require_label() for item in initial_test], + metadata={"split_metadata": {}}, + cache_namespace="fast_features", + windowing_policy={"strategy": "fixture"}, + noise_statistics={}, + ) + + plan_path = settings.tmp_folder / "prepared-training-fast.json" + with training_operation_scope(TrainingOperation(mode=TrainingMode.PREPARE_ONLY)) as state: + state.readiness = readiness + state.utterances = samples + with pytest.raises(OSError, match="publication interruption"): + _publish() + assert not plan_path.exists() + assert not list(plan_path.parent.glob(f".{plan_path.name}.*")) + + second = _cache_lookup() + assert second.cache_hit is True + assert compute_calls == 1 + _publish() + + assert plan_path.is_file() + assert orchestration.load_prepared_plan(plan_path).payload_path + + +@pytest.mark.parametrize("failure_phase", ["prepare", "publish", "unexpected"]) +def test_checked_backend_closes_once_for_every_failure_path(failure_phase: str) -> None: + closed = 0 + + class _Backend: + backend_id = "fixture" + feature_dim = 2 + + def close(self) -> None: + nonlocal closed + closed += 1 + + with pytest.raises(RuntimeError, match=failure_phase): + with training_operation_scope(TrainingOperation()): + orchestration.reuse_checked_backend( + backend_id="fixture", + model_id="model@revision", + device="cpu", + dtype="float32", + build=lambda: cast(FeatureBackend, _Backend()), + ) + raise RuntimeError(failure_phase) + assert closed == 1 diff --git a/tests/suites/unit/models/test_training_smoke_performance.py b/tests/suites/unit/models/test_training_smoke_performance.py new file mode 100644 index 0000000..685efda --- /dev/null +++ b/tests/suites/unit/models/test_training_smoke_performance.py @@ -0,0 +1,256 @@ +"""Bounded, offline evidence for training-readiness backend smoke checks.""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from pathlib import Path +from typing import overload + +import numpy as np +import pytest +import soundfile as sf +from numpy.typing import NDArray + +from ser._internal.config.schema import ( # noqa: TID251 + AudioReadConfig, + MediumRuntimeConfig, + RuntimeFlags, +) +from ser._internal.data.manifest import Utterance # noqa: TID251 +from ser._internal.models import training_orchestration # noqa: TID251 +from ser._internal.models.medium_noise_controls import ( # noqa: TID251 + apply_medium_noise_controls, +) +from ser._internal.models.training_readiness import ( # noqa: TID251 + TrainingOperation, + select_smoke_samples, +) +from ser._internal.pool import mean_std_pool, temporal_pooling_windows # noqa: TID251 +from ser._internal.repr import EncodedSequence, PoolingWindow # noqa: TID251 +from ser.config import AppConfig + + +class _CountingSequence(Sequence[Utterance]): + """Sequence that records full traversals without relying on timing.""" + + def __init__(self, rows: list[Utterance]) -> None: + self._rows = rows + self.iterations = 0 + self.visits = 0 + + def __len__(self) -> int: + return len(self._rows) + + @overload + def __getitem__(self, index: int) -> Utterance: ... + + @overload + def __getitem__(self, index: slice) -> list[Utterance]: ... + + def __getitem__(self, index: int | slice) -> Utterance | list[Utterance]: + return self._rows[index] + + def __iter__(self) -> Iterator[Utterance]: + self.iterations += 1 + for row in self._rows: + self.visits += 1 + yield row + + +class _StubBackend: + """Small deterministic sequence backend with observable inference inputs.""" + + backend_id = "stub-medium" + feature_dim = 3 + + def __init__(self) -> None: + self.encode_calls = 0 + self.decoded_inputs: list[NDArray[np.float32]] = [] + self.outputs: list[EncodedSequence] = [] + + def encode_sequence( + self, + audio: NDArray[np.float32], + sample_rate: int, + ) -> EncodedSequence: + self.encode_calls += 1 + decoded = np.asarray(audio, dtype=np.float32) + self.decoded_inputs.append(decoded.copy()) + chunks = np.array_split(decoded, 5) + embeddings = np.asarray( + [ + (float(chunk.mean()), float(chunk.std()), float(np.max(np.abs(chunk)))) + for chunk in chunks + ], + dtype=np.float32, + ) + frame_duration = (decoded.size / sample_rate) / len(chunks) + starts = np.arange(len(chunks), dtype=np.float64) * frame_duration + encoded = EncodedSequence( + embeddings=embeddings, + frame_start_seconds=starts, + frame_end_seconds=starts + frame_duration, + backend_id=self.backend_id, + ) + self.outputs.append(encoded) + return encoded + + def pool( + self, + encoded: EncodedSequence, + windows: Sequence[PoolingWindow], + ) -> NDArray[np.float64]: + return mean_std_pool(encoded, windows) + + +def _write_tiny_audio(path: Path, *, phase: float) -> None: + samples = np.arange(800, dtype=np.float32) + signal = 2.0 * np.sin((samples / 31.0) + phase) + stereo = np.column_stack((signal, signal * 0.25)) + subtype = "FLOAT" if path.suffix == ".wav" else "PCM_16" + sf.write(path, stereo, 8_000, subtype=subtype) + + +def _inventory(paths: dict[tuple[str, str], Path], size: int) -> list[Utterance]: + combinations = (("en", ".wav"), ("en", ".flac"), ("es", ".wav"), ("es", ".flac")) + return [ + Utterance( + schema_version=1, + sample_id=f"sample-{index:05d}", + corpus="fixture", + audio_path=paths[combinations[index % len(combinations)]], + label="calm" if index % 2 == 0 else "happy", + language=combinations[index % len(combinations)][0], + ) + for index in range(size) + ] + + +def test_readiness_selection_is_single_pass_capped_and_covers_format_language( + tmp_path: Path, +) -> None: + paths = { + (language, suffix): tmp_path / f"{language}-{suffix.removeprefix('.')}" f"{suffix}" + for language in ("en", "es") + for suffix in (".wav", ".flac") + } + tracked = _CountingSequence(_inventory(paths, 4_096)) + + selected = select_smoke_samples(tracked, cap=4) + + assert tracked.iterations == 1 + assert tracked.visits == len(tracked) + assert len(selected) == 4 + assert {(row.language, row.audio_path.suffix) for row in selected} == { + ("en", ".wav"), + ("en", ".flac"), + ("es", ".wav"), + ("es", ".flac"), + } + assert [row.sample_id for row in selected] == [ + row.sample_id for row in select_smoke_samples(list(reversed(tracked)), cap=4) + ] + + +def test_stub_backend_smoke_decodes_normalizes_and_round_trips_isolated_cache( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths = { + (language, suffix): tmp_path / f"{language}-{suffix.removeprefix('.')}{suffix}" + for language in ("en", "es") + for suffix in (".wav", ".flac") + } + for phase, path in enumerate(paths.values()): + _write_tiny_audio(path, phase=float(phase)) + samples = tuple(select_smoke_samples(_inventory(paths, 20_000), cap=4)) + backend = _StubBackend() + settings = AppConfig( + emotions={"01": "calm", "02": "happy"}, + tmp_folder=tmp_path / "tmp", + runtime_flags=RuntimeFlags(profile_pipeline=True, medium_profile=True), + audio_read=AudioReadConfig(max_retries=1, retry_delay_seconds=0.0), + medium_runtime=MediumRuntimeConfig( + pool_window_size_seconds=0.04, + pool_window_stride_seconds=0.02, + ), + ) + monkeypatch.setattr( + training_orchestration, + "_profile_smoke_runtime", + lambda _settings: ( + backend.backend_id, + "stub/model@revision", + "cpu", + "float32", + backend, + ), + ) + probe_cache = tmp_path / "isolated-smoke-cache" + + with training_orchestration.training_operation_scope(TrainingOperation()): + first = training_orchestration.run_selected_backend_smoke( + settings=settings, + samples=samples, + probe_cache_dir=probe_cache, + ) + with training_orchestration.training_operation_scope(TrainingOperation()): + second = training_orchestration.run_selected_backend_smoke( + settings=settings, + samples=samples, + probe_cache_dir=probe_cache, + ) + + assert backend.encode_calls == 4 + assert all(row.ndim == 1 and row.dtype == np.float32 for row in backend.decoded_inputs) + assert all( + np.all(np.isfinite(row)) and np.max(np.abs(row)) <= 1.0 for row in backend.decoded_inputs + ) + assert all(output.embeddings.shape == (5, backend.feature_dim) for output in backend.outputs) + assert all(np.all(np.isfinite(output.embeddings)) for output in backend.outputs) + assert {output.backend_id for output in backend.outputs} == {backend.backend_id} + assert first == second + assert first.attempted == first.succeeded == len(samples) == 4 + assert first.feature_dim == backend.feature_dim * 2 + assert first.cache_round_trip is True + assert (first.backend_id, first.model_id, first.device, first.dtype) == ( + backend.backend_id, + "stub/model@revision", + "cpu", + "float32", + ) + cache_paths = list((probe_cache / "embeddings").rglob("*.npz")) + assert len(cache_paths) == 4 + with np.load(cache_paths[0], allow_pickle=False) as payload: + assert str(np.asarray(payload["backend_id"]).item()) == backend.backend_id + + +def test_temporal_pooling_and_noise_controls_preserve_final_feature_dimension() -> None: + encoded = EncodedSequence( + embeddings=np.asarray( + [[0.0, 0.0], [0.0, 0.0], [1.0, -1.0], [2.0, -2.0], [3.0, -3.0]], + dtype=np.float32, + ), + frame_start_seconds=np.arange(5, dtype=np.float64) * 0.1, + frame_end_seconds=(np.arange(5, dtype=np.float64) + 1.0) * 0.1, + backend_id="stub-medium", + ) + windows = temporal_pooling_windows( + encoded, + window_size_seconds=0.2, + window_stride_seconds=0.1, + ) + pooled = mean_std_pool(encoded, windows) + controlled, stats = apply_medium_noise_controls( + pooled, + min_window_std=0.05, + max_windows_per_clip=2, + ) + + assert pooled.shape == (4, 4) + assert controlled.shape == (2, 4) + assert np.all(np.isfinite(controlled)) + assert stats.total_windows == 4 + assert stats.kept_windows == 2 + assert stats.dropped_low_std_windows >= 1 + assert stats.dropped_cap_windows >= 1 From 4d25479115f613687886d39da267f51636efcd6f Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:19:11 -0300 Subject: [PATCH 08/10] feat(cli): add dry-run, prepare-only, and prepared-plan modes Lets an expensive training run be validated before it starts and its preparation reused afterwards. --dry-run checks readiness and never fits, caches, or writes artifacts. --prepare-only materializes reusable features. --prepared-plan consumes a digest-validated plan. --repair is accepted only alongside the first two. Conflicting or non-training options now fail during argument validation, before any registry, backend, or feature work. Validation and policy failures exit 2 and unexpected failures exit 1 with a traceback, so a caller can distinguish "you asked for something impossible" from "we broke". The traceback is taken from the classified exception rather than from ambient exception state. --- ser/__main__.py | 161 +++++++++++++++++++++-- ser/_internal/runtime/commands.py | 26 +++- tests/suites/integration/cli/test_cli.py | 72 ++++++++++ 3 files changed, 247 insertions(+), 12 deletions(-) diff --git a/ser/__main__.py b/ser/__main__.py index e9fa58d..f770089 100644 --- a/ser/__main__.py +++ b/ser/__main__.py @@ -8,6 +8,8 @@ import time from collections.abc import Sequence from contextlib import AbstractContextManager, nullcontext +from dataclasses import replace +from pathlib import Path from typing import cast from dotenv import load_dotenv @@ -29,12 +31,18 @@ run_training_command, run_transcription_runtime_calibration_command, ) +from ser._internal.models.training_orchestration import ( + current_training_state, + training_operation_active, + training_operation_scope, +) +from ser._internal.models.training_readiness import TrainingMode, TrainingOperation from ser._internal.runtime.phase_timing import format_duration from ser._internal.utils.logger import configure_logging, get_logger from ser._internal.utils.subtitles import SUPPORTED_SUBTITLE_FORMATS from ser.config import AppConfig, reload_settings, settings_override from ser.diagnostics.domain import PreflightMode -from ser.profiles import ProfileName +from ser.profiles import ProfileName, resolve_profile_name from ser.runtime.contracts import SubtitleFormat logger: logging.Logger = get_logger("ser") @@ -133,6 +141,46 @@ def _build_main_parser(settings: AppConfig) -> argparse.ArgumentParser: action="store_true", help="Train the emotion classification model", ) + training_mode = parser.add_mutually_exclusive_group() + training_mode.add_argument( + "--dry-run", + action="store_true", + help=( + "Run bounded training readiness and backend smoke checks without fitting " + "or writing model artifacts." + ), + ) + training_mode.add_argument( + "--prepare-only", + action="store_true", + help="Validate and materialize reusable training features without fitting a classifier.", + ) + parser.add_argument( + "--prepared-plan", + type=str, + default=None, + help="Consume a digest-validated prepared training plan.", + ) + parser.add_argument( + "--repair", + action="store_true", + help="Enable allowlisted idempotent repairs during dry-run or preparation.", + ) + parser.add_argument( + "--dataset-recipe", + type=str, + default=None, + help=( + "Versioned dataset recipe JSON path or built-in 'research-v1'. " + "Training audits and routes manifests before feature extraction." + ), + ) + parser.add_argument( + "--strict-dataset-audit", + action=argparse.BooleanOptionalAction, + default=None, + help="Fail training on duplicate content, missing revisions/hashes, leakage, or empty classes.", + ) parser.add_argument( "--file", type=str, @@ -232,8 +280,7 @@ def _build_main_parser(settings: AppConfig) -> argparse.ArgumentParser: type=str, default="accurate,medium,accurate-research,fast", help=( - "Comma-separated profile list for calibration " - "(fast,medium,accurate,accurate-research)." + "Comma-separated profile list for calibration (fast,medium,accurate,accurate-research)." ), ) return parser @@ -246,6 +293,21 @@ def _settings_scope(active_settings: object) -> AbstractContextManager[object]: return nullcontext() +def _apply_dataset_recipe_override(settings: AppConfig, args: argparse.Namespace) -> AppConfig: + """Applies CLI dataset recipe/audit overrides to one immutable settings snapshot.""" + recipe_arg = args.dataset_recipe if isinstance(args.dataset_recipe, str) else None + strict_arg = args.strict_dataset_audit + if recipe_arg is None and strict_arg is None: + return settings + recipe = recipe_arg or settings.dataset.recipe + strict = ( + bool(strict_arg) + if isinstance(strict_arg, bool) + else True if recipe_arg is not None else settings.dataset.strict_audit + ) + return replace(settings, dataset=replace(settings.dataset, recipe=recipe, strict_audit=strict)) + + def _run_restricted_backend_gate( args: argparse.Namespace, *, active_settings: object ) -> int | None: @@ -297,7 +359,7 @@ def _run_calibration_or_exit(args: argparse.Namespace) -> None: logger.error( "%s", calibration_error.message, - exc_info=calibration_error.include_traceback, + exc_info=calibration_error.exc_info, ) sys.exit(calibration_error.exit_code) if calibration_result is None: @@ -320,10 +382,66 @@ def _run_calibration_or_exit(args: argparse.Namespace) -> None: sys.exit(0) -def _run_training_or_exit(*, active_settings: object) -> None: +def _validate_training_mode_args( + parser: argparse.ArgumentParser, + args: argparse.Namespace, +) -> None: + """Rejects train-only option misuse before any runtime or backend work.""" + has_train_only_option = bool( + args.dry_run or args.prepare_only or args.repair or args.prepared_plan is not None + ) + if has_train_only_option and not args.train: + parser.error("--dry-run, --prepare-only, --repair, and --prepared-plan require --train.") + if args.repair and not (args.dry_run or args.prepare_only): + parser.error("--repair is valid only with --dry-run or --prepare-only.") + if args.prepared_plan is not None and (args.dry_run or args.prepare_only): + parser.error("--prepared-plan is valid only for real training.") + + +def _training_operation_from_args(args: argparse.Namespace) -> TrainingOperation: + """Translates validated CLI flags into the typed orchestration contract.""" + mode = ( + TrainingMode.DRY_RUN + if args.dry_run + else TrainingMode.PREPARE_ONLY if args.prepare_only else TrainingMode.TRAIN + ) + prepared_plan = ( + Path(args.prepared_plan).expanduser() if isinstance(args.prepared_plan, str) else None + ) + return TrainingOperation(mode=mode, repair=bool(args.repair), prepared_plan=prepared_plan) + + +def _log_training_mode_start( + args: argparse.Namespace, + *, + active_settings: AppConfig, + preflight_mode: PreflightMode, +) -> None: + """Emits the first visible training-mode lifecycle event before preflight work.""" + operation = _training_operation_from_args(args) + logger.info( + "TRAIN_MODE_START mode=%s profile=%s preflight=%s repair=%s prepared_plan=%s", + operation.mode.value, + resolve_profile_name(active_settings), + preflight_mode, + operation.repair, + operation.prepared_plan is not None, + ) + + +def _run_training_or_exit( + args: argparse.Namespace | None = None, + *, + active_settings: object, +) -> None: """Runs training flow and exits with the appropriate status code.""" logger.info("Starting model training...") start_time = time.perf_counter() + operation = ( + current_training_state().operation + if training_operation_active() + else _training_operation_from_args(args) if args is not None else TrainingOperation() + ) disposition = run_training_command( settings=cast(AppConfig, active_settings), pipeline_builder=build_runtime_pipeline, @@ -332,11 +450,17 @@ def _run_training_or_exit(*, active_settings: object) -> None: logger.error( "%s", disposition.message, - exc_info=disposition.include_traceback, + exc_info=disposition.exc_info, ) sys.exit(disposition.exit_code) + completion_label = { + TrainingMode.DRY_RUN: "Training readiness dry run", + TrainingMode.PREPARE_ONLY: "Training preparation", + TrainingMode.TRAIN: "Training", + }[operation.mode] logger.info( - "Training completed in %s.", + "%s completed in %s.", + completion_label, format_duration(time.perf_counter() - start_time), ) sys.exit(0) @@ -366,7 +490,7 @@ def _run_inference_or_exit(args: argparse.Namespace, *, active_settings: object) logger.error( "%s", disposition.message, - exc_info=disposition.include_traceback, + exc_info=disposition.exc_info, ) sys.exit(disposition.exit_code) timeline_csv_path = getattr(execution, "timeline_csv_path", None) if execution else None @@ -393,6 +517,7 @@ def main() -> None: parser = _build_main_parser(settings) args: argparse.Namespace = parser.parse_args() + _validate_training_mode_args(parser, args) configure_logging(args.log_level) active_settings = apply_cli_profile_override( settings, @@ -402,8 +527,24 @@ def main() -> None: active_settings, disable_timeouts=bool(args.disable_timeouts), ) + active_settings = _apply_dataset_recipe_override(active_settings, args) preflight_mode = parse_preflight_mode(str(args.preflight)) - with _settings_scope(active_settings): + training_scope: AbstractContextManager[object] = ( + cast( + AbstractContextManager[object], + training_operation_scope(_training_operation_from_args(args)), + ) + if args.train + else nullcontext() + ) + with _settings_scope(active_settings), training_scope: + if args.train: + _log_training_mode_start( + args, + active_settings=active_settings, + preflight_mode=preflight_mode, + ) + restricted_exit_code = _run_restricted_backend_gate( args, active_settings=active_settings, @@ -423,7 +564,7 @@ def main() -> None: _run_calibration_or_exit(args) if args.train: - _run_training_or_exit(active_settings=active_settings) + _run_training_or_exit(args, active_settings=active_settings) _run_inference_or_exit(args, active_settings=active_settings) diff --git a/ser/_internal/runtime/commands.py b/ser/_internal/runtime/commands.py index e21062e..75999a1 100644 --- a/ser/_internal/runtime/commands.py +++ b/ser/_internal/runtime/commands.py @@ -3,8 +3,9 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path +from types import TracebackType from typing import TYPE_CHECKING from ser._internal.utils.subtitles import resolve_subtitle_export_request @@ -29,16 +30,34 @@ class WorkflowErrorDisposition: exit_code: int message: str include_traceback: bool = False + exception: Exception | None = field(default=None, repr=False, compare=False) + + @property + def exc_info(self) -> tuple[type[BaseException], BaseException, TracebackType | None] | bool: + """Returns explicit logging traceback data without consulting active exception state.""" + if not self.include_traceback or self.exception is None: + return False + return (type(self.exception), self.exception, self.exception.__traceback__) def classify_training_exception(err: Exception) -> WorkflowErrorDisposition: """Classifies one training exception into CLI logging/exit behavior.""" - if isinstance(err, RuntimeError): + from ser._internal.models.training_readiness import ( + PreparedPlanError, + QuarantineBudgetExceeded, + TrainingReadinessError, + ) + + if isinstance( + err, + (PreparedPlanError, QuarantineBudgetExceeded, TrainingReadinessError), + ): return WorkflowErrorDisposition(exit_code=2, message=str(err)) return WorkflowErrorDisposition( exit_code=1, message=f"Training workflow failed: {err}", include_traceback=True, + exception=err, ) @@ -109,11 +128,13 @@ def classify_inference_exception(err: Exception) -> WorkflowErrorDisposition: exit_code=3, message=f"Transcription failed: {err}", include_traceback=True, + exception=err, ) return WorkflowErrorDisposition( exit_code=1, message=f"Prediction workflow failed: {err}", include_traceback=True, + exception=err, ) @@ -273,6 +294,7 @@ def run_transcription_runtime_calibration_command( exit_code=1, message=f"Transcription runtime calibration failed: {err}", include_traceback=True, + exception=err, ), ) return (calibration_result, None) diff --git a/tests/suites/integration/cli/test_cli.py b/tests/suites/integration/cli/test_cli.py index 2ee39e4..24f42fd 100644 --- a/tests/suites/integration/cli/test_cli.py +++ b/tests/suites/integration/cli/test_cli.py @@ -1,5 +1,6 @@ """Behavior tests for CLI argument dispatch and exit semantics.""" +import logging from dataclasses import replace from pathlib import Path from types import SimpleNamespace @@ -13,6 +14,7 @@ from ser._internal.config.schema import profile_artifact_file_names from ser._internal.license_check import load_persisted_backend_consents from ser._internal.runtime.accurate_inference import AccurateRuntimeDependencyError +from ser._internal.runtime.commands import WorkflowErrorDisposition, classify_training_exception from ser._internal.runtime.medium_inference import MediumRuntimeDependencyError from ser._internal.runtime.registry import UnsupportedProfileError from ser.runtime.contracts import InferenceRequest @@ -339,6 +341,50 @@ def run_training(self) -> None: assert called["train"] is True +def test_cli_training_mode_start_logs_before_preflight( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Training mode should be visible before preflight can start long readiness work.""" + monkeypatch.setattr(cli, "load_dotenv", lambda: None) + monkeypatch.setattr( + cli, + "run_restricted_backend_cli_gate", + lambda **_kwargs: ((), None), + ) + base_settings = config_module.reload_settings() + monkeypatch.setattr( + cli, + "reload_settings", + lambda: replace( + base_settings, + runtime_flags=replace(base_settings.runtime_flags, profile_pipeline=True), + ), + ) + monkeypatch.setattr( + cli.sys, + "argv", + ["ser", "--train", "--profile", "medium", "--dry-run"], + ) + caplog.set_level(logging.INFO, logger="ser") + + def _preflight_gate(**_kwargs: object) -> tuple[tuple[tuple[str, str], ...], None]: + assert any("TRAIN_MODE_START mode=dry_run" in item.message for item in caplog.records) + return ((), None) + + class FakePipeline: + def run_training(self) -> None: + return None + + monkeypatch.setattr(cli, "run_startup_preflight_cli_gate", _preflight_gate) + monkeypatch.setattr(cli, "build_runtime_pipeline", lambda _settings: FakePipeline()) + + with pytest.raises(SystemExit) as exc_info: + cli.main() + + assert exc_info.value.code == 0 + + def test_cli_prediction_uses_runtime_pipeline_when_enabled( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -877,6 +923,32 @@ def _run_training_command(**_kwargs: object) -> object: assert captured == {"profile": 1, "timeout": 1, "train": 1} +def test_cli_training_failure_logs_preserved_traceback( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Training failures should log their real traceback outside the catch scope.""" + + def _failure_disposition() -> WorkflowErrorDisposition: + try: + raise OSError("invalid audio bytes") + except OSError as err: + return classify_training_exception(err) + + monkeypatch.setattr( + cli, + "run_training_command", + lambda **_kwargs: _failure_disposition(), + ) + + with caplog.at_level("ERROR"), pytest.raises(SystemExit) as exc_info: + cli._run_training_or_exit(active_settings=config_module.reload_settings()) + + assert exc_info.value.code == 1 + assert "OSError: invalid audio bytes" in caplog.text + assert "NoneType: None" not in caplog.text + + def test_cli_infer_path_calls_api_delegates_once( monkeypatch: pytest.MonkeyPatch, ) -> None: From 76f9abb450421508cb4ff116b69a5895cb565145 Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:19:11 -0300 Subject: [PATCH 09/10] feat(diagnostics): include training readiness in preflight Surfaces registry and training readiness findings through `ser doctor` and the training preflight, so the checks are reachable before committing to a run. --preflight off still disables only the general startup preflight; it never disables the training readiness contract itself. --- ser/_internal/api/diagnostics.py | 6 ++ ser/_internal/diagnostics/command.py | 3 +- ser/_internal/diagnostics/service.py | 105 ++++++++++++++++++++++++--- 3 files changed, 104 insertions(+), 10 deletions(-) diff --git a/ser/_internal/api/diagnostics.py b/ser/_internal/api/diagnostics.py index 3546296..ac17c6e 100644 --- a/ser/_internal/api/diagnostics.py +++ b/ser/_internal/api/diagnostics.py @@ -83,6 +83,8 @@ def run_startup_preflight( *, settings: AppConfig, include_transcription_checks: bool, + include_dataset_registry_checks: bool = False, + include_training_readiness: bool = False, ) -> DiagnosticReport: """Runs one startup preflight diagnostics check suite.""" from ser._internal.diagnostics.service import _run_startup_preflight @@ -90,6 +92,8 @@ def run_startup_preflight( return _run_startup_preflight( settings=settings, include_transcription_checks=include_transcription_checks, + include_dataset_registry_checks=include_dataset_registry_checks, + include_training_readiness=include_training_readiness, ) @@ -142,6 +146,8 @@ def run_startup_preflight_cli_gate( no_transcript=no_transcript, calibrate_transcription_runtime=calibrate_transcription_runtime, ), + include_dataset_registry_checks=train_requested, + include_training_readiness=train_requested, ) doctor_command = resolve_doctor_command(profile=profile) fail_preflight = should_fail_preflight(report=report, mode=mode) diff --git a/ser/_internal/diagnostics/command.py b/ser/_internal/diagnostics/command.py index 0899533..63c7312 100644 --- a/ser/_internal/diagnostics/command.py +++ b/ser/_internal/diagnostics/command.py @@ -34,7 +34,7 @@ def run_doctor_command(argv: list[str], *, settings: AppConfig | None = None) -> "--profile", choices=_PROFILE_CHOICES, default=None, - help=("Profile context for diagnostics " "(fast, medium, accurate, accurate-research)."), + help=("Profile context for diagnostics (fast, medium, accurate, accurate-research)."), ) parser.add_argument( "--format", @@ -68,6 +68,7 @@ def run_doctor_command(argv: list[str], *, settings: AppConfig | None = None) -> settings=active_settings, include_transcription_checks=not bool(args.skip_transcription_checks), include_noise_findings=bool(args.include_noise_findings), + include_training_readiness=True, ) formatted_output = ( format_report_json(report) if args.format == "json" else format_report_text(report) diff --git a/ser/_internal/diagnostics/service.py b/ser/_internal/diagnostics/service.py index 76aa499..0ca29e9 100644 --- a/ser/_internal/diagnostics/service.py +++ b/ser/_internal/diagnostics/service.py @@ -3,7 +3,9 @@ from __future__ import annotations import json +import logging import shutil +import time from dataclasses import replace from typing import Literal, cast @@ -19,6 +21,7 @@ DEFAULT_MPS_LOW_MEMORY_THRESHOLD_GB, resolve_transcription_runtime_policy, ) +from ser._internal.utils.logger import get_logger from ser._internal.utils.transcription_compat import ( FASTER_WHISPER_OPENMP_CONFLICT_ISSUE_CODE, resolve_transcription_compatibility_lane, @@ -27,10 +30,12 @@ from ser.diagnostics.domain import ( DiagnosticFinding, DiagnosticReport, + DiagnosticSeverity, PreflightMode, ) from ser.profiles import ProfileName, get_profile_catalog, resolve_profile_name +logger: logging.Logger = get_logger(__name__) _SUPPORTED_PREFLIGHT_MODES: frozenset[str] = frozenset({"off", "warn", "strict"}) @@ -65,13 +70,42 @@ def _run_startup_preflight( *, settings: AppConfig, include_transcription_checks: bool, + include_dataset_registry_checks: bool = False, + include_training_readiness: bool = False, ) -> DiagnosticReport: """Runs one fast startup preflight report for command execution gating.""" - findings = _run_checks( - settings=settings, - include_transcription_checks=include_transcription_checks, - include_noise_findings=False, - include_lane_info=False, + started_at = time.perf_counter() + logger.info( + "PREFLIGHT_START include_transcription=%s include_registry=%s " + "include_training_readiness=%s", + include_transcription_checks, + include_dataset_registry_checks, + include_training_readiness, + ) + try: + findings = _run_checks( + settings=settings, + include_transcription_checks=include_transcription_checks, + include_noise_findings=False, + include_lane_info=False, + include_dataset_registry_checks=include_dataset_registry_checks, + ) + if include_training_readiness: + findings.extend(_check_complete_training_readiness(settings=settings)) + except Exception: + logger.exception( + "PREFLIGHT_DONE status=FAIL elapsed=%.1fs", + time.perf_counter() - started_at, + ) + raise + status = ( + "FAIL" if any(finding.blocking for finding in findings) else "WARN" if findings else "PASS" + ) + logger.info( + "PREFLIGHT_DONE status=%s findings=%d elapsed=%.1fs", + status, + len(findings), + time.perf_counter() - started_at, ) return DiagnosticReport(findings=tuple(findings)) @@ -81,6 +115,7 @@ def run_doctor_diagnostics( settings: AppConfig, include_transcription_checks: bool, include_noise_findings: bool, + include_training_readiness: bool = False, ) -> DiagnosticReport: """Runs one comprehensive diagnostics report for interactive troubleshooting.""" findings = _run_checks( @@ -90,9 +125,62 @@ def run_doctor_diagnostics( include_lane_info=True, include_dataset_registry_checks=True, ) + if include_training_readiness: + findings.extend(_check_complete_training_readiness(settings=settings)) return DiagnosticReport(findings=tuple(findings)) +def _check_complete_training_readiness(*, settings: AppConfig) -> tuple[DiagnosticFinding, ...]: + """Runs the same bounded checker/backend smoke used by every training entrypoint.""" + from ser._internal.data.data_loader import load_utterances + from ser._internal.models.training_orchestration import ( + ensure_entrypoint_readiness, + training_operation_active, + training_operation_scope, + ) + from ser._internal.models.training_readiness import ( + FailureSeverity, + ReadinessReport, + TrainingMode, + TrainingOperation, + TrainingReadinessError, + ) + + def _run() -> ReadinessReport: + try: + report, _ = ensure_entrypoint_readiness( + settings=settings, + load_utterances=lambda: load_utterances( + settings=settings, + allow_prepare=False, + ), + ) + except TrainingReadinessError as error: + return error.report + return report + + if training_operation_active(): + report = _run() + else: + with training_operation_scope(TrainingOperation(mode=TrainingMode.DRY_RUN)): + report = _run() + severity_map: dict[FailureSeverity, DiagnosticSeverity] = { + FailureSeverity.DEBUG: "info", + FailureSeverity.INFO: "info", + FailureSeverity.WARNING: "warning", + FailureSeverity.ERROR: "error", + } + return tuple( + DiagnosticFinding( + code=f"training_readiness_{finding.reason_code.value}", + severity=severity_map[finding.severity], + message=finding.message, + blocking=finding.blocking, + ) + for finding in report.findings + ) + + def should_fail_preflight(*, report: DiagnosticReport, mode: PreflightMode) -> bool: """Returns whether one startup preflight report should block command execution.""" if mode == "off": @@ -107,7 +195,7 @@ def format_report_text(report: DiagnosticReport) -> str: counts = report.counts_by_severity() lines: list[str] = [ "SER diagnostics report", - ("summary: " f"info={counts['info']} warning={counts['warning']} error={counts['error']}"), + (f"summary: info={counts['info']} warning={counts['warning']} error={counts['error']}"), ] if not report.findings: lines.append("status: ok (no findings)") @@ -139,7 +227,7 @@ def format_report_brief( counts = report.counts_by_severity() lines: list[str] = [ "SER diagnostics preflight summary", - ("summary: " f"info={counts['info']} warning={counts['warning']} error={counts['error']}"), + (f"summary: info={counts['info']} warning={counts['warning']} error={counts['error']}"), ] if not report.findings: lines.append("status: ok (no findings)") @@ -250,8 +338,7 @@ def _check_ffmpeg_binary() -> tuple[DiagnosticFinding, ...]: code="ffmpeg_binary_missing", severity="error", message=( - "ffmpeg executable was not found on PATH; transcription workflows " - "require ffmpeg." + "ffmpeg executable was not found on PATH; transcription workflows require ffmpeg." ), remediation=("Install ffmpeg and rerun diagnostics (`brew install ffmpeg` on macOS).",), blocking=True, From 6843dc36c94417241f7fe3895a65c39bf00cf50a Mon Sep 17 00:00:00 2001 From: Juan Sugg Date: Wed, 15 Jul 2026 18:19:11 -0300 Subject: [PATCH 10/10] docs(training): document readiness, recipes, and containment Documents the readiness contract and its side-effect boundaries, the repair allowlist, the fault-containment budgets, and the versioned recipe workflow. --- CHANGELOG.md | 28 +++++++++++++ docs/training-readiness.md | 82 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 docs/training-readiness.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 44f1736..463cf6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,29 @@ after its first published distribution. inverse-square-root class mass, and bounded seeded window selection. - Feature backends resolve and expose the exact model revision they loaded, and accept a `repository@revision` model identifier to pin it. +- Manifest schema v2: categorical emotion is now optional, and rows can carry + normalized VAD, social attitude, binary affect, language, transcript, per + target source/confidence, speaker/session identity, native split, segment + bounds, normalized-PCM SHA-256, dataset revision, license, and provenance. + Schema v1 manifests remain readable. +- Versioned dataset recipes selected with `--dataset-recipe` or + `SER_DATASET_RECIPE`, including the built-in `research-v1`. Recipes declare + per-corpus label and task routing, and are audited into a leakage-safe split + ledger with reproducible recipe, manifest, and ledger digests. + Strictness is controlled by `--strict-dataset-audit` / `SER_STRICT_DATASET_AUDIT`. +- Model artifact envelope v3, persisting recipe and split-ledger digests, model + revision, task heads, sampling policy, seed, and evaluation summary. Envelope + v2 artifacts still load, and the public eight-emotion inference schema is + unchanged. +- Mandatory training readiness contract on every CLI and library training + entrypoint, with `--dry-run`, `--prepare-only`, `--prepared-plan`, and + `--repair` modes documented in `docs/training-readiness.md`. +- Bias-aware quarantine budgets through `SER_MAX_FAILED_FILES`, + `SER_MAX_FAILED_FILE_RATIO_PER_CORPUS`, `SER_MAX_FAILED_FILE_RATIO_PER_CLASS`, + `SER_MAX_FAILURES_PER_REASON`, `SER_MIN_REMAINING_PER_CLASS_SPLIT`, and + `SER_STRICT_QUARANTINE`. +- `SER_DEV_SIZE` (default `0.10`) reserves a deterministic, speaker-isolated + development partition before windowing. ### Changed @@ -27,6 +50,9 @@ after its first published distribution. cause. Both subclass `OSError`, so existing handling keeps working. - Preparing CREMA-D now requires Git and Git LFS up front and fails with an actionable message when either is missing. +- Training validation and policy failures now exit `2`; unexpected internal + failures continue to exit `1` with a traceback. +- `ser doctor` and the training preflight now include training readiness checks. - README Python API guidance now directs workflow users to `ser.api`. ### Fixed @@ -36,6 +62,8 @@ after its first published distribution. decode failure during feature extraction. - CREMA-D audio is validated before manifest registration, so an incomplete checkout fails during dataset preparation rather than mid-training. +- Speaker partitioning during dataset splitting is now linear rather than + quadratic, so large inventories no longer appear to hang. - Import lint path collection now works on shells without `mapfile`, such as the Bash 3.2 that ships with macOS. diff --git a/docs/training-readiness.md b/docs/training-readiness.md new file mode 100644 index 0000000..7979d0d --- /dev/null +++ b/docs/training-readiness.md @@ -0,0 +1,82 @@ +# Training readiness, preparation, and fault containment + +Every CLI and library training entrypoint runs the same mandatory readiness contract before a +classifier is created or fitted. `--preflight off` disables only the general startup preflight; it +does not disable training readiness. + +## Commands + +```bash +ser --train --profile medium --dry-run +ser --train --profile medium --dry-run --repair +ser --train --profile medium --prepare-only +ser --train --profile medium --prepare-only --repair +ser --train --profile medium --prepared-plan ~/.cache/ser/tmp/prepared-training-medium.json +``` + +- `--dry-run` validates configuration, registry/media integrity, split feasibility, write paths, + disk/file-descriptor capacity, and a deterministic bounded real-backend sample. It writes + `training-readiness-.json` under `SER_TMP_DIR` and never creates/fits a classifier, + populates production embedding caches, writes a quarantine JSONL ledger, or writes model + artifacts. Quarantine findings and audit timestamps remain in the atomic readiness report. +- `--prepare-only` first runs readiness, then performs complete feature/cache preparation. It + atomically writes a non-pickle NPZ feature payload followed by + `prepared-training-.json`. A failed/interrupted payload publication never produces a + ready plan. Re-running is safe and reuses valid embedding-cache entries. +- `--prepared-plan` verifies the canonical plan digest, feature-payload digest, profile/settings, + backend/model revision, device/dtype, registry, manifests, media fingerprints, recipe, split and + quarantine ledgers, code-owned cache namespace/version, and content-bound keys. Backend plans + require the exact commit resolved by the checked backend; an unverified mutable model identifier + cannot produce or reuse a plan. Plans and payloads also bind actual train/dev/test sample and + window ledgers plus included/quarantined/dropped dispositions, shapes, dtypes, labels, and current + effective sample IDs. Any mismatch exits with validation status `2`; valid plans bypass completed + feature extraction. +- `--repair` is accepted only with `--dry-run` or `--prepare-only`. Repairs are explicit, + idempotent, recorded in the readiness report, and revalidated. The allowlist creates missing + application-owned directories, removes abandoned application staging/probe files, invalidates + corrupt NPZ entries only inside fixed application-owned cache namespaces, rebuilds missing + manifests from intact registered sources, + and hydrates compatible Git LFS checkouts. A revision-pinned Hugging Face model may be + redownloaded only when `SER_TRAINING_REPAIR_ALLOW_NETWORK=1` explicitly permits network access. + It never deletes datasets, changes labels or thresholds, accepts licenses/consents, or + overwrites a valid model artifact. + +Conflicting or non-training options fail during argument validation, before registry, backend, or +feature work. Validation/policy failures exit `2`; unexpected internal failures exit `1` with a +traceback. + +## Fault containment + +Failures carry a stable scope, reason, severity, and disposition. Git LFS pointers, invalid +configuration/manifests, leakage, insufficient class support, non-finite backend output, resource +shortages, and unknown exceptions always abort. Medium and accurate profiles may quarantine only +known sample-local decode failures or an exact missing media path proven inside a registered root. +Generic/cache/model `OSError` and unproven missing paths abort. Every proposed exclusion must satisfy +the absolute, global, per-corpus, per-class, per-reason, and minimum remaining class/split budgets. +Strict quarantine mode disables these exclusions. + +The compatibility ratio `SER_MAX_FAILED_FILE_RATIO` remains supported. Bias-aware controls are +available through `SER_MAX_FAILED_FILES`, `SER_MAX_FAILED_FILE_RATIO_PER_CORPUS`, +`SER_MAX_FAILED_FILE_RATIO_PER_CLASS`, `SER_MAX_FAILURES_PER_REASON`, +`SER_MIN_REMAINING_PER_CLASS_SPLIT`, and `SER_STRICT_QUARANTINE`. + +`SER_DEV_SIZE` (default `0.10`) reserves a deterministic, speaker-isolated development partition +before windowing. Together with `SER_TEST_SIZE`, it must leave a non-empty training ratio; readiness +validates minimum per-class support independently across train, dev, and test. + +Quarantines are written as deterministic, bounded JSONL under `SER_TMP_DIR`; paths are represented +by SHA-256 digests. Source audio is never mutated. Silent/low-variance medium windows continue to +use the existing deterministic window-noise controls, and corrupt embedding-cache entries continue +to be atomically invalidated and recomputed. Optional secure artifact failure remains advisory only +after the primary artifact is valid; primary model/report persistence remains fatal. + +## Side effects and sizing + +Readiness uses a fixed sample cap and linear media inventory passes. Write/rename probes use unique +names and are removed in `finally`. Preparation concurrency remains bounded by configured worker +limits. Progress logs report processed/total, cache activity, quarantines, elapsed time, and a +credible ETA at bounded intervals. Backend smoke and permitted network repairs require hard-deadline +support; unsupported worker/platform execution is rejected rather than running unbounded. Reports, +quarantine ledgers, feature payloads, and plans use atomic publication; no partial document is +treated as ready. Quarantine ledger reuse identity excludes wall-clock fields while JSONL/report +timestamps remain available for audit.