feat(validate): story_end check, scene-spec av_sync anchors, layout budgets

AGENTS.md

@@ -47,7 +47,7 @@ Commands registered on the **`docgen`** CLI include:
 - **`image-generate`** — render scene-spec **image elements** (`image:` + `prompt:` boxes) via the OpenAI Images API into the bundle (also runs for missing assets inside `generate-all`).
 - **`manim`** — render Manim scenes declared in config.
 - **`compose`** — mux narration audio with visual sources via ffmpeg.
-- **`validate`** / **`validate --pre-push`** — drift, narration lint, Manim hints, **`timing_sync`**, **`av_sync`** (soft), **`subject_beat_coverage`** (declarative specs vs narration topic beats; hard fail when enabled), and related checks.
+- **`validate`** / **`validate --pre-push`** — drift, narration lint, Manim hints, **`timing_sync`**, **`story_end`** (last paced reveal vs audio end; hard fail), **`av_sync`** (soft; prefers scene-spec labels as OCR anchors), **`subject_beat_coverage`** (declarative specs vs narration topic beats; hard fail when enabled), and related checks.
 - **`lint`** — narration lint helper.
 - **`narration-generate`** — LLM-assisted narration from hints and repo context.
 - **`scene-spec-generate`** — LLM emits declarative **`*.scene.yaml`**; enforces frame budget + **subject-beat coverage** (dwell OK; cover topic shifts; reject invented labels).

README.md

@@ -54,7 +54,8 @@ If you still need the legacy behaviour, pin a pre-removal commit
   segments, with a freeze-tail guard.
 - **Validation** — A/V drift, freeze ratio, OCR error scan, layout, narration lint,
   Manim scene lint, **timing_sync** (stale `timing.json` vs regenerated mp3 —
-  hard fail), and **av_sync** (OCR check that spoken anchor keywords appear on
+  hard fail), **story_end** (paced visual story finishes long before narration —
+  hard fail), and **av_sync** (OCR check that scene-spec label anchors appear on
   screen near their spoken time — soft warning).
 - **GitHub Pages** — auto-generate `index.html`, deploy workflow, LFS rules,
   `.gitignore`.
@@ -180,9 +181,14 @@ validation:
     enabled: true
     max_tail_gap_sec: 3.0    # mp3 may run this much past the last transcribed word
     max_end_overrun_sec: 1.0 # transcript may extend this far past the mp3
+  story_end:                 # last paced reveal vs audio end (hard fail in --pre-push)
+    enabled: true
+    max_early_sec: 40.0      # idle after last paced box
+    max_early_ratio: 0.45    # and idle / audio_end (both must exceed to fail)
   av_sync:                   # OCR anchor check (soft warning in --pre-push)
     enabled: true
     tolerance_sec: 3.0
+    prefer_scene_spec_labels: true  # OCR anchors from paced box labels when specs exist
     visual_types: [manim]    # only check types with on-screen text
 
 timestamps:

src/docgen/av_sync.py

@@ -26,6 +26,15 @@ class AVSyncReport:
     passed: bool = True
 
 
+def _ocr_keyword_from_label(label: str) -> str | None:
+    """Pick a distinctive token from a scene-spec label for OCR substring match."""
+    tokens = re.findall(r"[A-Za-z][A-Za-z0-9-]{2,}", label or "")
+    if not tokens:
+        return None
+    # Prefer longer tokens (OCR noise is worse on short words).
+    return max(tokens, key=len)
+
+
 class AVSyncValidator:
     def __init__(self, config: Config) -> None:
         self.config = config
@@ -116,18 +125,65 @@ def _get_anchors(self, seg_id: str, ts_data: dict[str, Any]) -> list[SyncAnchor]
                 for a in configured
             ]
 
-        # Auto-extract: nouns from transcript words that are >5 chars
         words = ts_data.get("words", [])
+        if not isinstance(words, list):
+            words = []
+
+        if self.sync_cfg.get("prefer_scene_spec_labels", True):
+            from_spec = self._anchors_from_scene_specs(seg_id, words)
+            if from_spec:
+                return from_spec
+
+        # Fallback: nouns from transcript words that are >5 chars
         seen: set[str] = set()
         anchors: list[SyncAnchor] = []
-        min_anchors = self.sync_cfg.get("min_anchors_per_segment", 2)
+        min_anchors = int(self.sync_cfg.get("min_anchors_per_segment", 2))
 
         for w in words:
+            if not isinstance(w, dict):
+                continue
             word = re.sub(r"[^a-zA-Z]", "", w.get("word", ""))
             if len(word) > 5 and word.lower() not in seen:
                 seen.add(word.lower())
                 anchors.append(SyncAnchor(keyword=word, spoken_at=w.get("start", 0)))
                 if len(anchors) >= min_anchors * 2:
                     break
 
-        return anchors[:max(min_anchors, 3)]
+        return anchors[: max(min_anchors, 3)]
+
+    def _anchors_from_scene_specs(
+        self, seg_id: str, words: list[Any]
+    ) -> list[SyncAnchor]:
+        """Build OCR anchors from paced scene-spec labels (on-screen text)."""
+        if not words:
+            return []
+
+        from docgen.scene_retime import list_scene_spec_paths
+        from docgen.scene_spec import iter_paced_label_anchors, load_scene_spec
+
+        paths = list_scene_spec_paths(self.config, segment_id=seg_id)
+        if not paths:
+            return []
+
+        min_anchors = int(self.sync_cfg.get("min_anchors_per_segment", 2))
+        max_anchors = int(self.sync_cfg.get("max_anchors_per_segment", 8))
+        max_anchors = max(min_anchors, max_anchors)
+
+        seen: set[str] = set()
+        anchors: list[SyncAnchor] = []
+        word_dicts = [w for w in words if isinstance(w, dict)]
+
+        for path in paths:
+            try:
+                spec = load_scene_spec(path)
+            except Exception:
+                continue
+            for label, spoken_at in iter_paced_label_anchors(spec, word_dicts):
+                keyword = _ocr_keyword_from_label(label)
+                if not keyword or keyword.lower() in seen:
+                    continue
+                seen.add(keyword.lower())
+                anchors.append(SyncAnchor(keyword=keyword, spoken_at=spoken_at))
+                if len(anchors) >= max_anchors:
+                    return anchors
+        return anchors

src/docgen/config.py

@@ -275,6 +275,9 @@ def av_sync_config(self) -> dict[str, Any]:
             "enabled": True,
             "tolerance_sec": 3.0,
             "min_anchors_per_segment": 2,
+            "max_anchors_per_segment": 8,
+            # Prefer scene-spec box labels as OCR anchors when specs exist.
+            "prefer_scene_spec_labels": True,
             # Only OCR-anchor these visual types (on-screen text expected).
             "visual_types": ["manim"],
         }
@@ -298,6 +301,23 @@ def timing_sync_config(self) -> dict[str, Any]:
         defaults.update(self.raw.get("validation", {}).get("timing_sync", {}))
         return defaults
 
+    @property
+    def story_end_config(self) -> dict[str, Any]:
+        """Visual story finished early vs narration (``docgen validate`` ``story_end``).
+
+        Compares the last paced scene-spec reveal (label→``wait_word`` start) to
+        the audio/transcript end. Fails when idle time after the last reveal
+        exceeds **both** ``max_early_sec`` and ``max_early_ratio`` × audio end
+        (hard fail in ``--pre-push``, like ``timing_sync``).
+        """
+        defaults: dict[str, Any] = {
+            "enabled": True,
+            "max_early_sec": 40.0,
+            "max_early_ratio": 0.45,
+        }
+        defaults.update(self.raw.get("validation", {}).get("story_end", {}))
+        return defaults
+
     @property
     def narration_lint_config(self) -> dict[str, Any]:
         defaults: dict[str, Any] = {

src/docgen/scene_spec.py

@@ -593,6 +593,65 @@ def pacing_violations(spec: dict[str, Any], *, words_present: bool) -> list[str]
     return issues
 
 
+def iter_paced_label_anchors(
+    spec: dict[str, Any],
+    words: list[dict[str, Any]],
+) -> list[tuple[str, float]]:
+    """Return ``(label, spoken_start)`` for paced boxes after fail-closed sync.
+
+    Used by validate ``story_end`` and ``av_sync`` OCR anchoring.
+    """
+    if not isinstance(words, list) or not words:
+        return []
+    synced = sync_row_labels_to_whisper_words(spec, words, overwrite=True)
+    out: list[tuple[str, float]] = []
+    for rows in _spec_pages_rows(synced):
+        for row in rows:
+            if not isinstance(row, dict) or _pace_none(row):
+                continue
+            boxes = row.get("boxes")
+            if not isinstance(boxes, list):
+                continue
+            for box in boxes:
+                if not isinstance(box, dict) or _pace_none(box):
+                    continue
+                label = str(box.get("label", "")).strip()
+                ww = box.get("wait_word")
+                if not label or ww is None:
+                    continue
+                try:
+                    wi = int(ww)
+                except (TypeError, ValueError):
+                    continue
+                if wi < 0 or wi >= len(words):
+                    continue
+                w = words[wi]
+                if not isinstance(w, dict):
+                    continue
+                try:
+                    t = float(w.get("start", 0.0))
+                except (TypeError, ValueError):
+                    continue
+                out.append((label, t))
+    return out
+
+
+def last_paced_reveal_time(
+    spec: dict[str, Any],
+    words: list[dict[str, Any]],
+) -> float | None:
+    """Wall-clock ``start`` of the latest paced box reveal, or ``None`` if none.
+
+    Re-derives ``wait_word`` from labels (fail-closed sync) then takes the maximum
+    word ``start`` among boxes that are not ``pace: none``. Used by validate
+    ``story_end`` to detect boards that finish long before the narration ends.
+    """
+    anchors = iter_paced_label_anchors(spec, words)
+    if not anchors:
+        return None
+    return max(t for _, t in anchors)
+
+
 def upgrade_wait_segments_to_wait_words(
     spec: dict[str, Any],
     words: list[dict[str, Any]],

src/docgen/scene_spec_generate.py

@@ -241,17 +241,27 @@ def build_scene_spec_user_message(
 
     parts.append("")
     parts.append("--- FRAME / LAYOUT BUDGET (plan every page; scene-spec-generate rejects overflow) ---")
+    horiz_safe = FRAME_WIDTH - 1.0
+    budget_default = layout_stack_budget(
+        {"font_size": 36}, {"first_row_title_buff": 0.5}
+    )
+    budget_compact = layout_stack_budget(
+        {"font_size": 32}, {"first_row_title_buff": 0.45}
+    )
     parts.append(
-        f"Dogfood Manim frame ≈ {FRAME_WIDTH} × {FRAME_HEIGHT} Manim units. "
-        "Per page vertical cost = sum over rows of max(box height in row) + (n_rows - 1) * row_gap. "
-        "That total must stay at or below the budget implied by your title.font_size and layout.first_row_title_buff. "
-        "Per row horizontal cost = sum(box widths) + (n_boxes - 1) * column_gap; keep ≤ ~13."
+        f"Frame ≈ {FRAME_WIDTH:.2f} × {FRAME_HEIGHT:.2f} Manim units. "
+        f"Horizontal safe width ≈ {horiz_safe:.2f} u "
+        "(sum of box widths + (n_boxes-1)*column_gap per row must stay ≤ this)."
     )
     parts.append(
-        f"Reference max stack heights: "
-        f"title 36 + first_row_title_buff 0.5 → ≈ {layout_stack_budget({'font_size': 36}, {'first_row_title_buff': 0.5}):.2f} u; "
-        f"title 32 + buff 0.45 → ≈ {layout_stack_budget({'font_size': 32}, {'first_row_title_buff': 0.45}):.2f} u. "
-        "Recompute if you change those fields."
+        "**Vertical stack budgets** (use these numbers unless you change "
+        "title.font_size / layout.first_row_title_buff):\n"
+        f"  • Default font_size=36, first_row_title_buff=0.5 → "
+        f"max stack height ≈ {budget_default:.2f} u\n"
+        f"  • Compact font_size=32, first_row_title_buff=0.45 → "
+        f"max stack height ≈ {budget_compact:.2f} u\n"
+        "Per page: sum(max box height per row) + (n_rows-1)*row_gap ≤ that budget. "
+        "When you would exceed it, spill to another page (do not shrink/cram)."
     )
     return "\n".join(parts)
 

src/docgen/validate.py

@@ -296,6 +296,7 @@ def validate_segment(
             report.checks.append(CheckResult("recording_exists", False, [f"No recording for {seg_id}"]))
 
         report.checks.append(self._check_timing_sync(seg_id))
+        report.checks.append(self._check_story_end(seg_id))
         report.checks.append(self._check_narration_lint(seg_id))
         if self.config.visual_map.get(seg_id, {}).get("type") == "manim":
             report.checks.append(self._check_manim_scene_lint())
@@ -739,6 +740,94 @@ def _check_timing_sync(self, seg_id: str) -> CheckResult:
             )
         return CheckResult("timing_sync", True, [detail])
 
+    def _check_story_end(self, seg_id: str) -> CheckResult:
+        """Fail when the paced visual story finishes long before the narration ends.
+
+        Muxed recordings can still match mp3 length (compose freezes the last frame)
+        while the diagram finished early. Uses scene-spec label→``wait_word`` starts
+        vs audio/transcript end. Hard fail in ``--pre-push`` (not soft like ``av_sync``).
+        """
+        se_cfg = self.config.story_end_config
+        if not se_cfg.get("enabled", True):
+            return CheckResult("story_end", True, ["validation.story_end disabled (skipped)"])
+
+        is_manim = self.config.visual_map.get(seg_id, {}).get("type") == "manim"
+        if not is_manim:
+            return CheckResult("story_end", True, ["non-manim (skipped)"])
+
+        from docgen.scene_retime import list_scene_spec_paths
+        from docgen.scene_spec import last_paced_reveal_time, load_scene_spec
+
+        paths = list_scene_spec_paths(self.config, segment_id=seg_id)
+        if not paths:
+            return CheckResult(
+                "story_end", True, ["No animations/specs/*.scene.yaml (skipped)"]
+            )
+
+        block = self._load_timing_block(seg_id)
+        if block is None:
+            return CheckResult(
+                "story_end",
+                True,
+                ["No timing.json entry (skipped) — run `docgen timestamps`"],
+            )
+        words = block.get("words")
+        if not isinstance(words, list) or not words:
+            return CheckResult(
+                "story_end", True, ["No timing words (skipped)"]
+            )
+
+        last_reveal: float | None = None
+        for path in paths:
+            try:
+                spec = load_scene_spec(path)
+            except Exception as exc:
+                return CheckResult(
+                    "story_end",
+                    False,
+                    [f"Cannot load scene spec {path.name}: {exc}"],
+                )
+            t = last_paced_reveal_time(spec, words)
+            if t is not None and (last_reveal is None or t > last_reveal):
+                last_reveal = t
+
+        if last_reveal is None:
+            return CheckResult(
+                "story_end",
+                True,
+                ["No paced reveals in scene spec (skipped)"],
+            )
+
+        audio = self._find_audio(seg_id)
+        audio_end = self._probe_media_duration(audio) if audio and not _is_lfs_pointer(audio) else None
+        transcript_end = self._timing_last_end(block)
+        # Prefer audio duration; fall back to transcript end when probe fails.
+        end_t = audio_end if audio_end is not None else transcript_end
+        if end_t is None or end_t <= 0:
+            return CheckResult("story_end", True, ["Cannot determine audio/transcript end (skipped)"])
+
+        early_idle = end_t - last_reveal
+        max_early_sec = float(se_cfg.get("max_early_sec", 40.0))
+        max_early_ratio = float(se_cfg.get("max_early_ratio", 0.45))
+        early_ratio = early_idle / end_t if end_t > 0 else 0.0
+        detail = (
+            f"last_paced_reveal={last_reveal:.2f}s audio_end={end_t:.2f}s "
+            f"early_idle={early_idle:.2f}s ({early_ratio:.0%}) "
+            f"(max_early_sec={max_early_sec}, max_early_ratio={max_early_ratio})"
+        )
+        if early_idle > max_early_sec and early_ratio > max_early_ratio:
+            return CheckResult(
+                "story_end",
+                False,
+                [
+                    detail,
+                    "Visual story finishes long before narration ends — boxes race then freeze. "
+                    "Add paced labels for later narration beats, or run "
+                    "`docgen scene-spec-generate` / `scene-compile --retime` after timestamps.",
+                ],
+            )
+        return CheckResult("story_end", True, [detail])
+
     def _check_av_sync(self, seg_id: str, rec: Path) -> CheckResult:
         """OCR anchor check: spoken keywords should be visible on screen near their spoken time.