feat(flows): draft paper structure from full page text in bounded windows

contexts/design/mind/retrieval.md

@@ -54,7 +54,7 @@ flowchart TD
     subgraph FLW["flows — PaperFlow(cfg).build(input), pure processing"]
         OPEN["fetch + parse the input"]
         OUT["outline signals (deterministic)"]
-        DRAFT["draft structuring agent (model, private draft)"]
+        DRAFT["windowed draft structuring agent (model, private draft)"]
         BUILD["mint ids/links, read node text from cited pages, validate; self-contained PaperStructureTree (+ as_of / provenance)"]
     end
     subgraph LIB["library — dump / load only"]
@@ -76,7 +76,7 @@ flowchart TD
 | Owner | Responsibility |
 |---|---|
 | `quantmind.preprocess` | Emit deterministic outline signals (heading candidates, table-of-contents pages, printed-to-physical page offset) from a parsed document. No LLM calls. |
-| `quantmind.flows` (`PaperFlow`) | A **config-bound** flow: `PaperFlow(cfg)` binds the settings; `build(input)` fetches, parses, runs one draft-structuring agent, then calls the knowledge constructor and returns a **self-contained** `PaperStructureTree`. The cfg *type* selects the knowledge shape (`PaperStructureCfg` → tree today). No persistence, no retrieval, no library. |
+| `quantmind.flows` (`PaperFlow`) | A **config-bound** flow: `PaperFlow(cfg)` binds the settings; `build(input)` fetches, parses, drafts the hierarchy from full page text in character-bounded windows (one chained draft-structuring call per window, each extending the prior draft), then calls the knowledge constructor and returns a **self-contained** `PaperStructureTree`. The cfg *type* selects the knowledge shape (`PaperStructureCfg` → tree today). No persistence, no retrieval, no library. |
 | `quantmind.knowledge` | Own the `StructureTree` structural base and the source-bound `PaperStructureTree` artifact. `from_draft` mints identity, resolves page citations, **and populates each node's `content` from the exact source pages**, then runs the integrity gate. The artifact is a complete value. |
 | `quantmind.library` | Dump a self-contained tree and load it back unchanged (`put` / `open_structure`). A tree is an **independent** artifact: its library need not contain a chunk set, and loading it never depends on refilling text from another artifact. |
 | `quantmind.mind` | `AgenticRetriever(cfg)` binds the strategy config; `retrieve(tree, question)` reasons over one explicit tree value and returns evidence values with content already in them. It does **not** take or bind a library. |

examples/flows/paper_structure_windowed.py

@@ -0,0 +1,57 @@
+"""Draft a structure tree from full page text in bounded windows.
+
+``PaperFlow(PaperStructureCfg()).build`` reads every page complete: pages are
+packed into character-bounded windows (``window_chars`` per model call, with
+``window_overlap_pages`` shared pages for continuity), and a document larger
+than one window is drafted across chained calls that each extend the prior
+draft. Dense pages therefore keep their lower-page section starts and body
+prose visible to the drafting model.
+
+``page_text_chars`` stays available as an explicit per-page clip for cost
+control on sparse inputs (short pages, a clean table of contents); the
+``None`` default sends full pages.
+
+Running this end to end needs network access (a model provider). The example
+is written so it imports and type-checks offline.
+"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+from quantmind.configs import PaperStructureCfg
+from quantmind.configs.paper import LocalFilePath
+from quantmind.flows import PaperFlow
+
+
+async def main(pdf_path: Path) -> None:
+    """Build one windowed full-text structure tree for a local PDF."""
+    # Defaults draft from full page text: ~80k chars per window, one page of
+    # overlap between consecutive windows, and no per-page clipping.
+    flow = PaperFlow(PaperStructureCfg(model="gpt-5.6-luna"))
+    tree = await flow.build(LocalFilePath(path=pdf_path))
+
+    producer = tree.producer
+    print("orchestration:", producer.orchestration)
+    print("window_chars:", producer.window_chars)
+    print("window_overlap_pages:", producer.window_overlap_pages)
+    print("page_text_chars:", producer.page_text_chars)
+    for node in tree.nodes.values():
+        pages = [c.page for c in node.citations if c.page is not None]
+        print(f"{node.title} — pages {min(pages)}-{max(pages)}")
+
+    # Sparse inputs can trade fidelity for cost with an explicit clip; the
+    # producer records the policy, so both trees version independently.
+    clipped_flow = PaperFlow(
+        PaperStructureCfg(model="gpt-5.6-luna", page_text_chars=1_200)
+    )
+    clipped_tree = await clipped_flow.build(LocalFilePath(path=pdf_path))
+    print("clipped tree id differs:", clipped_tree.id != tree.id)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        raise SystemExit(
+            "usage: python examples/flows/paper_structure_windowed.py paper.pdf"
+        )
+    asyncio.run(main(Path(sys.argv[1])))

quantmind/configs/structure.py

@@ -11,12 +11,20 @@ class PaperStructureCfg(BaseFlowCfg):
     ``PaperFlow.build`` dispatches on the cfg **type**: constructing
     ``PaperFlow`` with a ``PaperStructureCfg`` selects the self-contained
     ``PaperStructureTree`` shape.
+
+    Drafting reads full page text in character-bounded windows
+    (``window_chars`` per model call, ``window_overlap_pages`` shared pages
+    between consecutive windows). ``page_text_chars`` is an optional per-page
+    clip for cost control on sparse inputs; the ``None`` default sends each
+    page complete.
     """
 
     model: str = "gpt-5.6-luna"
-    prompt_version: str = "paper-structure-v2"
+    prompt_version: str = "paper-structure-v3"
     instructions: str | None = None
-    page_text_chars: int = Field(default=1_200, ge=80)
+    page_text_chars: int | None = Field(default=None, ge=80)
+    window_chars: int = Field(default=80_000, ge=2_000)
+    window_overlap_pages: int = Field(default=1, ge=0)
     max_output_tokens: int = Field(default=4_096, gt=0)
     max_depth: int = Field(default=6, ge=1)
     max_nodes: int = Field(default=128, ge=1)

quantmind/flows/paper/__init__.py

@@ -11,9 +11,10 @@
 one config-bound flow produces every paper shape:
 
 - ``PaperStructureCfg`` selects the self-contained ``PaperStructureTree`` shape
-  (fetch + parse, deterministic outline signals, one draft-structuring agent,
-  then the knowledge-layer ``from_draft`` constructor that mints identity and
-  populates each leaf node's page-cited text).
+  (fetch + parse, deterministic outline signals, windowed full-page-text draft
+  structuring — one chained agent call per character-bounded window — then the
+  knowledge-layer ``from_draft`` constructor that mints identity and populates
+  each leaf node's page-cited text).
 - ``PaperSemanticCfg`` selects the source-first chunk/summary shape
   (``PaperSemanticResult``): fetch + parse, page-aware chunking, then a bounded
   map-reduce summary whose citations the knowledge layer resolves.
@@ -70,6 +71,7 @@
     _summary_instructions_hash,
 )
 from quantmind.flows.paper._structure import (
+    _STRUCTURE_ORCHESTRATION,
     PaperStructureError,
     _AgentsPaperStructureProvider,
     _PaperStructureProvider,
@@ -206,10 +208,11 @@ async def build(self, input: PaperInput) -> _ResultT:
         Dispatches on the bound cfg **type**:
 
         - ``PaperStructureCfg`` runs the structure pipeline (fetch + parse,
-          deterministic outline signals, one draft-structuring agent, then the
-          knowledge-layer constructor that mints identity, resolves page
-          citations, and populates each leaf node's ``content``), returning a
-          self-contained ``PaperStructureTree``.
+          deterministic outline signals, a draft-structuring agent reading
+          full page text in character-bounded windows — one chained call per
+          window — then the knowledge-layer constructor that mints identity,
+          resolves page citations, and populates each leaf node's
+          ``content``), returning a self-contained ``PaperStructureTree``.
         - ``PaperSemanticCfg`` runs the source-first chunk/summary pipeline (fetch +
           parse, page-aware chunking, bounded map-reduce summary), returning a
           ``PaperSemanticResult``.
@@ -256,8 +259,11 @@ async def _build_structure(
         producer = PaperStructureProducer(
             model=cfg.model,
             prompt_version=cfg.prompt_version,
+            orchestration=_STRUCTURE_ORCHESTRATION,
             instructions_hash=_structure_instructions_hash(cfg),
             page_text_chars=cfg.page_text_chars,
+            window_chars=cfg.window_chars,
+            window_overlap_pages=cfg.window_overlap_pages,
             max_output_tokens=cfg.max_output_tokens,
             max_depth=cfg.max_depth,
             max_nodes=cfg.max_nodes,

quantmind/flows/paper/_structure.py

@@ -1,10 +1,10 @@
-"""Single-pass draft structuring for an exact paper source revision."""
+"""Windowed full-text draft structuring for an exact paper source revision."""
 
 import asyncio
 import hashlib
 import json
 from dataclasses import replace
-from typing import Any, Protocol
+from typing import Any, Literal, Protocol
 
 from agents import Agent, ModelSettings
 
@@ -18,11 +18,20 @@
     run_structured,
 )
 
+_STRUCTURE_ORCHESTRATION: Literal["windowed-v1"] = "windowed-v1"
+
+_QUALITY_ORDER = {"low": 0, "medium": 1, "high": 2}
+
 _STRUCTURE_INSTRUCTIONS = """\
 Act as a paper structure specialist. Return one hierarchy draft and a quality
 rating. Use only the supplied outline signals and ordered physical-page text.
 Every node must name one inclusive physical-page span; a parent must include
-all physical pages included by its children. The root must cover every page.
+all physical pages included by its children. The payload covers one window of
+consecutive pages and names the document's full page range. When a prior
+draft is supplied as draft_so_far, extend or revise it with evidence from the
+window's pages and return the complete updated hierarchy, keeping earlier
+sections unless the new pages contradict them. The returned root must cover
+every page read so far; after the final window that is every document page.
 Use titles and concise summaries for reasoning. Do not invent UUIDs, parent
 links, citations, source text, or canonical identity. If the evidence does not
 support a reliable hierarchy, set quality to low so code can build a safe flat
@@ -35,7 +44,7 @@ class PaperStructureError(RuntimeError):
 
 
 class _PaperStructureProvider(Protocol):
-    """Test seam and production boundary for one structure draft call."""
+    """Test seam and production boundary for one structure draft."""
 
     async def structure(
         self,
@@ -65,7 +74,9 @@ def _structure_instructions_hash(cfg: PaperStructureCfg) -> str:
             "max_nodes": cfg.max_nodes,
             "max_output_tokens": cfg.max_output_tokens,
             "page_text_chars": cfg.page_text_chars,
-            "orchestration": "single-pass-v1",
+            "window_chars": cfg.window_chars,
+            "window_overlap_pages": cfg.window_overlap_pages,
+            "orchestration": _STRUCTURE_ORCHESTRATION,
         },
         ensure_ascii=False,
         separators=(",", ":"),
@@ -83,13 +94,78 @@ def _structure_model_settings(cfg: PaperStructureCfg) -> ModelSettings:
     )
 
 
-def _structure_payload(
-    signals: OutlineSignals,
+def _page_payloads(
     source: PaperSourceRevision,
     cfg: PaperStructureCfg,
+) -> tuple[dict[str, Any], ...]:
+    """Project parsed pages into prompt entries, clipping only when asked."""
+    return tuple(
+        {
+            "page_number": page.page_number,
+            "text": (
+                page.text
+                if cfg.page_text_chars is None
+                else page.text[: cfg.page_text_chars]
+            ),
+        }
+        for page in source.parsed.pages
+    )
+
+
+def _window_pages(
+    pages: tuple[dict[str, Any], ...],
+    *,
+    window_chars: int,
+    overlap_pages: int,
+) -> tuple[tuple[dict[str, Any], ...], ...]:
+    """Split ordered page entries into character-bounded page windows.
+
+    Pages are packed greedily until ``window_chars`` is reached; a page is
+    never split, so an oversized page forms its own window. Consecutive
+    windows share ``overlap_pages`` trailing pages for continuity, and every
+    window starts at least one page after its predecessor so packing always
+    terminates.
+    """
+    windows: list[tuple[dict[str, Any], ...]] = []
+    start = 0
+    while start < len(pages):
+        end = start
+        used = 0
+        while end < len(pages):
+            page_chars = len(pages[end]["text"])
+            if end > start and used + page_chars > window_chars:
+                break
+            used += page_chars
+            end += 1
+        windows.append(tuple(pages[start:end]))
+        if end >= len(pages):
+            break
+        start = max(end - overlap_pages, start + 1)
+    return tuple(windows)
+
+
+def _structure_payload(
+    signals: OutlineSignals,
+    pages: tuple[dict[str, Any], ...],
+    window: tuple[dict[str, Any], ...],
+    *,
+    window_index: int,
+    window_total: int,
+    draft_so_far: PaperStructureTreeDraft | None,
 ) -> str:
     return json.dumps(
         {
+            "document": {
+                "first_page": pages[0]["page_number"],
+                "last_page": pages[-1]["page_number"],
+                "page_count": len(pages),
+            },
+            "window": {
+                "index": window_index + 1,
+                "total": window_total,
+                "start_page": window[0]["page_number"],
+                "end_page": window[-1]["page_number"],
+            },
             "outline": {
                 "table_of_contents_pages": signals.table_of_contents_pages,
                 "printed_page_offset": signals.printed_page_offset,
@@ -102,20 +178,27 @@ def _structure_payload(
                     for heading in signals.headings
                 ],
             },
-            "pages": [
-                {
-                    "page_number": page.page_number,
-                    "text": page.text[: cfg.page_text_chars],
-                }
-                for page in source.parsed.pages
-            ],
+            "draft_so_far": (
+                None
+                if draft_so_far is None
+                else draft_so_far.root.model_dump(mode="json")
+            ),
+            "pages": list(window),
         },
         ensure_ascii=False,
     )
 
 
 class _AgentsPaperStructureProvider:
-    """Run one structured-output agent over deterministic outline signals."""
+    """Draft one hierarchy from full page text in character-bounded windows.
+
+    Every window carries complete page text (optionally clipped by
+    ``cfg.page_text_chars``); a document larger than ``cfg.window_chars``
+    is drafted across several model calls, each extending the prior draft.
+    ``cfg.timeout_seconds`` bounds each model call. The returned draft keeps
+    the worst quality rating seen across windows, so one unreliable window
+    routes the whole document to the deterministic flat fallback.
+    """
 
     async def structure(
         self,
@@ -124,8 +207,46 @@ async def structure(
         *,
         cfg: PaperStructureCfg,
     ) -> PaperStructureTreeDraft:
-        payload = _structure_payload(signals, source, cfg)
+        pages = _page_payloads(source, cfg)
+        if not pages:
+            raise PaperStructureError(
+                "paper structure drafting requires at least one parsed page"
+            )
+        windows = _window_pages(
+            pages,
+            window_chars=cfg.window_chars,
+            overlap_pages=cfg.window_overlap_pages,
+        )
+        draft: PaperStructureTreeDraft | None = None
+        worst_quality: Literal["low", "medium", "high"] = "high"
+        for window_index, window in enumerate(windows):
+            payload = _structure_payload(
+                signals,
+                pages,
+                window,
+                window_index=window_index,
+                window_total=len(windows),
+                draft_so_far=draft,
+            )
+            draft = await self._draft_window(payload, cfg)
+            if _QUALITY_ORDER[draft.quality] < _QUALITY_ORDER[worst_quality]:
+                worst_quality = draft.quality
+        if draft is None:  # pragma: no cover - guarded by the pages check
+            raise PaperStructureError(
+                "paper structure drafting produced no draft"
+            )
+        if draft.quality != worst_quality:
+            draft = PaperStructureTreeDraft(
+                root=draft.root,
+                quality=worst_quality,
+            )
+        return draft
 
+    async def _draft_window(
+        self,
+        payload: str,
+        cfg: PaperStructureCfg,
+    ) -> PaperStructureTreeDraft:
         def build_agent(json_object: bool) -> Agent[Any]:
             instructions = _structure_instructions(cfg)
             model_settings = _structure_model_settings(cfg)

quantmind/knowledge/paper.py

@@ -705,15 +705,25 @@ class PaperStructureTreeDraft(BaseModel):
 
 
 class PaperStructureProducer(BaseModel):
-    """Exact model, prompt, page-input, and bounds used to structure a paper."""
+    """Exact model, prompt, page-input, and bounds used to structure a paper.
+
+    ``orchestration`` names the draft input policy: ``single-pass-v1`` sent
+    every page once, clipped to ``page_text_chars``; ``windowed-v1`` sends
+    full page text in character-bounded windows (``window_chars`` per call,
+    ``window_overlap_pages`` shared pages between consecutive windows) with an
+    optional per-page clip. The window fields are ``None`` on artifacts
+    produced by the single-pass policy.
+    """
 
     model_config = ConfigDict(extra="forbid", frozen=True)
 
     model: str
     prompt_version: str
-    orchestration: Literal["single-pass-v1"] = "single-pass-v1"
+    orchestration: Literal["single-pass-v1", "windowed-v1"] = "single-pass-v1"
     instructions_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
-    page_text_chars: int = Field(ge=80)
+    page_text_chars: int | None = Field(ge=80)
+    window_chars: int | None = Field(default=None, ge=2_000)
+    window_overlap_pages: int | None = Field(default=None, ge=0)
     max_output_tokens: int = Field(gt=0)
     max_depth: int = Field(ge=1)
     max_nodes: int = Field(ge=1)