diff --git a/.gitignore b/.gitignore index 8247e23..cc32695 100644 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,10 @@ runs/ *.pth *.safetensors +# Generated Raw-preserving smoke metadata. +smoke_raw_preserving_v1.json +raw_preserving_smoke_*.jsonl + # Credentials and local access configuration. *.pem *.key diff --git a/README.md b/README.md index 457c3e7..1b076fa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,72 @@ # Borderless Table Structuring Lab +Research infrastructure for auditable borderless-table structure recognition, +Canonical Table supervision, safe Raw MinerU refinement, and independent +Explicit and LoRA candidate routes. + +This repository is designed as the long-lived project home. The initial +revision contains the data-engineering and safety-integration layers only. It +does **not** contain model weights, training payloads, terminal benchmark pages, +Customer50 artifacts, or per-sample terminal predictions. + +## Table of contents + +- [Research objective](#research-objective) +- [Current repository scope](#current-repository-scope) +- [Repository layout](#repository-layout) +- [System design](#system-design) +- [Data strategy](#data-strategy) +- [Installation](#installation) +- [Tests](#tests) +- [Reproducibility and evidence](#reproducibility-and-evidence) +- [Collaboration workflow](#collaboration-workflow) +- [Roadmap](#roadmap) +- [Governance and licensing](#governance-and-licensing) + +## Research objective + +The project targets table-quality improvement under the OmniDocBench document +parsing protocol while preserving the Raw MinerU document baseline. The core +engineering principle is selective, auditable table correction: + +1. Raw MinerU remains the default output. +2. The Explicit route may propose minimal topology-only corrections with Raw + OCR text frozen. +3. The LoRA route may propose one complete, table-only Canonical Table state. +4. Both routes pass through the same legality, token-preservation, geometry, + provenance, expected-gain, assembly, and exact-Raw-rollback controls. +5. Unsafe or unsupported candidates are rejected without modifying Raw. + +The target of Table TEDS above 95 is an engineering objective, not a guaranteed +unobserved result. Public benchmark-aware development and independent terminal +generalization must be reported separately. + +## Current repository scope + +Included in the first revision: + +- Canonical Table normalization and structural label utilities. +- Direct-state and order-invariant target compilation. +- Candidate-integrity checks. +- Shared fail-closed validation and deterministic Raw rollback. +- Explicit topology-only and LoRA complete-table candidate interfaces. +- Synthetic unit fixtures and regression tests. +- Canonical record schema. +- Evidence Cards and the active execution contract. +- Dataset governance, storage, reproducibility, and collaborator handoff + documentation. +- Public OTSL normalization and fixed-denominator paired-metric utilities. +- Synthetic-data provenance guidance and manifest validation. + +Explicitly excluded: + +- Model implementations, adapters, checkpoints, or weights. +- Full training corpora or rendered sample payloads. +- Formal20k source records and compiled record payloads. +- Customer50 content. +- OmniDocBench pages, crops, annotations, recognized strings, coordinates, + HTML, LaTeX, page identifiers, or Gold records. +- Per-sample terminal predictions or case-selection artifacts. Research on recovering table structure from weak or absent visual boundaries. The repository brings together canonical table representations, controlled data generation, explicit topology modeling, and parameter-efficient @@ -171,6 +238,8 @@ python -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip python -m pip install -e '.[dev]' +# Add Pillow-backed terminal-blind rendering and perceptual-overlap audits. +python -m pip install -e '.[dev,synthesis]' ``` Run the data-free test suite: @@ -208,7 +277,11 @@ documentation needed to reproduce them. For details, see [Reproducibility](docs/REPRODUCIBILITY_2026.08.12.1.md) and [Dataset Storage and Sharing](docs/corpus/DATASET_STORAGE_AND_SHARING_2026.08.12.1.md). -## Collaboration +Detailed requirements are documented in +[REPRODUCIBILITY.md](docs/REPRODUCIBILITY.md). The bounded KEEP-majority pair +contract, exact-Raw baseline, streaming family audit, and fail-closed perceptual +overlap gate are described in +[RAW_PRESERVING_SMOKE.md](docs/RAW_PRESERVING_SMOKE.md). The two model tracks share representations and evaluation but keep model code and ablations independent. Suggested branch prefixes are: diff --git a/docs/RAW_PRESERVING_SMOKE.md b/docs/RAW_PRESERVING_SMOKE.md new file mode 100644 index 0000000..5e16510 --- /dev/null +++ b/docs/RAW_PRESERVING_SMOKE.md @@ -0,0 +1,55 @@ +# Raw-preserving bounded smoke + +The first smoke is intentionally small and deterministic: + +- 256 base table groups; +- 10 counterfactual records per group; +- all authorized phenomenon families represented; +- 90% `KEEP_RAW` and 10% `ACCEPT_EDIT` oracle labels after Gold-only labeling; +- no document, template, content, renderer, or seed family crosses roles. + +The smoke must report requested, generated, passed, quarantined, and failed +counts. It must also check canonical legality, complete grid coverage, OCR token +ownership, geometry, direct Gold recompilation, deterministic replay, and exact +/perceptual/structure/text/geometry overlap. Family identifiers are not proof of +visual isolation: cross-role images are screened with 9-by-8 dHash and confirmed +with 32-by-32 pHash. A confirmed perceptual hit blocks the corpus rather than +silently removing records or retuning the threshold after holdout inspection. +Large-corpus audits must stream records or retain only compact group summaries; +they must not require all Raw, Gold, and candidate objects in memory. + +## Oracle action contract + +The offline labeler receives `raw_record`, `candidate_record`, and +`gold_record`. It emits `KEEP_RAW` unless all of the following hold: + +1. the candidate passes the shared safety validator; +2. the candidate has strictly more exact Gold cells than Raw; +3. candidate text agreement is not lower than Raw; +4. candidate geometry coverage is not lower than Raw; +5. non-table page state is unchanged. + +A Raw state that exactly matches Gold is always `KEEP_RAW`, even when a +candidate is also valid. Ties and partial improvements are `KEEP_RAW`. + +Gold and all Gold-derived values are offline-only fields. The runtime selector +view contains only the image, Raw record, candidate record, and observable +candidate-vs-Raw differences. + +## Initial distribution + +The frozen first distribution is 90% KEEP and 10% ACCEPT: + +- 35% Raw-correct with a legal harmful over-edit candidate; +- 25% Raw-near-correct with inflation, over-split, or over-merge candidates; +- 15% Raw-bad with a tied, partial, or trade-off candidate; +- 15% identity candidates; +- 5% Raw-bad with a clearly better local candidate; +- 5% Raw-bad with a clearly better complete candidate. + +The first two KEEP-heavy curricula are 95/5 and then 90/10. A K0 all-KEEP +execution baseline must pass before any selector training. Selector-only audits +use a frozen controlled candidate bank and report routing separately from +candidate generation. In particular, an offline Gold-derived positive candidate +may measure takeover logic, but it is never runtime evidence and cannot support +a claim about a deployable candidate generator. diff --git a/docs/SYNTHETIC_DATA_PROVENANCE.md b/docs/SYNTHETIC_DATA_PROVENANCE.md new file mode 100644 index 0000000..d9b4aed --- /dev/null +++ b/docs/SYNTHETIC_DATA_PROVENANCE.md @@ -0,0 +1,41 @@ +# Synthetic corpus provenance + +This repository does not store rendered data, benchmark pages, model outputs, or +training payloads. The approved synthetic packages are shared separately. + +## SynthFin v3.4 + +SynthFin v3.4 is a locally authored renderer for financial-report-style table +pages. It does not copy pixels or annotations from FinTabNet, PubTables, +OmniDocBench, CNInfo, or customer documents. Its vocabulary, company-like +names, financial values, page prose, table topology, and metadata are generated +from local templates and seeded random draws. + +The renderer supports Chinese and English financial statements, borderless and +ruled tables, multi-row headers, row and column spans, dense pages, multiple +tables per page, narrative context, optional formulas, resolution ladders, and +light JPEG capture effects. The v3.4 correction avoids drawing ruled segments +through the interior of merged cells. The ground-truth structure is generated +from the same table state used by the renderer and is validated by round-trip +checks before use. + +The shared package contains only synthetic images and its manifest. It does not +contain the upstream public datasets used as external references, model +weights, terminal benchmark content, or prediction files. + +## invoice-synthetic-v1 + +The invoice package is fully self-generated. It uses local templates for +invoice-like layouts, random company names, dates, identifiers, addresses, +items, tax rates, totals, and Chinese amount-in-words strings. Each text draw is +recorded at generation time, so the visible-text annotations and rendered +images share one source of truth. The package does not use external invoice +images or scanned documents as backgrounds. + +## Scope and limitations + +These packages are research synthetic data, not official benchmark data and +not a claim of real-domain performance. They are intended for debugging, +controlled ablations, and reproducibility of rendering/label contracts. Users +should inspect the applicable license and sharing terms before redistributing +modified versions. diff --git a/pyproject.toml b/pyproject.toml index 0208e4b..a03bb60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,9 @@ dependencies = [ dev = [ "pytest>=7.4,<9", ] +synthesis = [ + "Pillow>=10,<13", +] [tool.setuptools] package-dir = {"" = "src"} diff --git a/schemas/raw_preserving_pair_v1_schema.json b/schemas/raw_preserving_pair_v1_schema.json new file mode 100644 index 0000000..914a811 --- /dev/null +++ b/schemas/raw_preserving_pair_v1_schema.json @@ -0,0 +1,134 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "mpr-tsr/raw-preserving-pair-v1", + "title": "Raw-Preserving Candidate Pair", + "type": "object", + "required": [ + "schema_version", + "sample_id", + "pair_group_id", + "role", + "image", + "raw_record", + "gold_record", + "candidate_record", + "oracle_decision", + "provenance" + ], + "properties": { + "schema_version": {"const": "mpr-tsr/raw-preserving-pair-v1"}, + "sample_id": {"type": "string", "minLength": 1}, + "pair_group_id": {"type": "string", "minLength": 1}, + "role": {"enum": ["train", "development", "holdout", "terminal"]}, + "image": { + "type": "object", + "required": [ + "relative_path", + "sha256", + "source_document_hash", + "table_index", + "image_size" + ], + "properties": { + "relative_path": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?![A-Za-z]:)(?!.*(?:^|/)\\.\\.(?:/|$))[^\\\\]+$" + }, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "source_document_hash": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "table_index": {"type": "integer", "minimum": 0}, + "image_size": { + "type": "array", + "prefixItems": [ + {"type": "integer", "minimum": 1}, + {"type": "integer", "minimum": 1} + ], + "items": false + } + }, + "additionalProperties": false + }, + "raw_record": {"type": "object"}, + "gold_record": {"type": "object"}, + "candidate_record": {"type": ["object", "null"]}, + "oracle_decision": { + "type": "object", + "required": ["action", "reason", "raw_good"], + "properties": { + "action": {"enum": ["KEEP_RAW", "ACCEPT_EDIT"]}, + "reason": {"type": "string", "minLength": 1}, + "raw_good": {"type": "boolean"} + }, + "additionalProperties": true + }, + "phenomenon_tags": {"type": "array", "items": {"type": "string"}}, + "operator": {"type": "object"}, + "provenance": { + "type": "object", + "required": [ + "generator", + "generator_version", + "generation_seed", + "terminal_data_used", + "document_cluster_id", + "template_family_id", + "content_family_id", + "renderer_family_id" + ], + "properties": { + "generator": {"type": "string", "minLength": 1}, + "generator_version": {"type": "string", "minLength": 1}, + "generator_policy": {"const": "terminal_blind_v1"}, + "generation_seed": {"type": "integer", "minimum": 0}, + "terminal_data_used": {"const": false}, + "document_cluster_id": {"type": "string", "minLength": 1}, + "template_family_id": {"type": "string", "minLength": 1}, + "content_family_id": {"type": "string", "minLength": 1}, + "renderer_family_id": {"type": "string", "minLength": 1}, + "family_plan_entry_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "source_dataset": {"type": "string", "minLength": 1}, + "license": {"type": "string", "minLength": 1}, + "font": { + "type": "object", + "required": ["source", "file_name", "sha256", "redistributed"], + "properties": { + "source": {"type": "string", "minLength": 1}, + "file_name": {"type": "string", "minLength": 1}, + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "redistributed": {"const": false} + }, + "additionalProperties": false + } + }, + "additionalProperties": true + } + }, + "allOf": [ + { + "if": { + "properties": { + "provenance": { + "required": ["generator_policy"], + "properties": { + "generator_policy": {"const": "terminal_blind_v1"} + } + } + } + }, + "then": { + "properties": { + "provenance": { + "required": [ + "family_plan_entry_sha256", + "source_dataset", + "license", + "font" + ] + } + } + } + } + ], + "additionalProperties": false +} diff --git a/scripts/audit_raw_preserving_overlap.py b/scripts/audit_raw_preserving_overlap.py new file mode 100644 index 0000000..1b92721 --- /dev/null +++ b/scripts/audit_raw_preserving_overlap.py @@ -0,0 +1,262 @@ +from __future__ import annotations + +import argparse +import hashlib +import itertools +import json +import math +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable, Iterator + +from PIL import Image + + +FINGERPRINT_KINDS = ("image_sha256", "structure", "text", "geometry", "source") + + +def _load(path: Path) -> Iterator[dict[str, Any]]: + with path.open(encoding="utf-8") as handle: + for line in handle: + if line.strip(): + yield json.loads(line) + + +def _hash(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _dhash(path: Path) -> int: + with Image.open(path) as image: + resized = image.convert("L").resize((9, 8)) + get_pixels = getattr(resized, "get_flattened_data", resized.getdata) + pixels = list(get_pixels()) + bits = 0 + for row in range(8): + start = row * 9 + for col in range(8): + bits = (bits << 1) | int(pixels[start + col] > pixels[start + col + 1]) + return bits + + +def _phash(path: Path) -> int: + size = 32 + low = 8 + with Image.open(path) as image: + resized = image.convert("L").resize((size, size), Image.Resampling.LANCZOS) + get_pixels = getattr(resized, "get_flattened_data", resized.getdata) + pixels = [float(value) for value in get_pixels()] + cosines = [ + [math.cos((2 * position + 1) * frequency * math.pi / (2 * size)) for position in range(size)] + for frequency in range(low) + ] + coefficients = [] + for vertical in range(low): + for horizontal in range(low): + coefficient = 0.0 + for row in range(size): + vertical_weight = cosines[vertical][row] + offset = row * size + coefficient += vertical_weight * sum( + pixels[offset + col] * cosines[horizontal][col] + for col in range(size) + ) + coefficients.append(coefficient) + values = sorted(coefficients[1:]) + median = values[len(values) // 2] + bits = 0 + for value in coefficients[1:]: + bits = (bits << 1) | int(value > median) + return bits + + +def _hamming(left: int, right: int) -> int: + return (left ^ right).bit_count() + + +def _gold_fingerprints(record: dict[str, Any]) -> dict[str, str]: + table = record["gold_record"]["canonical_table"] + cells = sorted( + table.get("cells", []), + key=lambda cell: ( + int(cell.get("row", 0)), + int(cell.get("col", 0)), + str(cell.get("cell_id", "")), + ), + ) + structure = { + "rows": table.get("rows"), + "cols": table.get("cols"), + "cells": [ + { + "row": cell.get("row"), + "col": cell.get("col"), + "rowspan": cell.get("rowspan"), + "colspan": cell.get("colspan"), + "tag": cell.get("tag"), + } + for cell in cells + ], + } + text = [str(cell.get("text", "")).strip() for cell in cells] + geometry = [cell.get("bbox", cell.get("geometry", {}).get("bbox")) for cell in cells] + return { + "structure": _hash(structure), + "text": _hash(text), + "geometry": _hash(geometry), + "source": str(record["image"]["source_document_hash"]), + } + + +def audit(records: Iterable[dict[str, Any]], image_root: Path, perceptual_distance: int) -> dict[str, Any]: + groups: dict[str, dict[str, Any]] = {} + record_count = 0 + for record in records: + record_count += 1 + group_id = str(record["pair_group_id"]) + fingerprints = _gold_fingerprints(record) + if group_id not in groups: + groups[group_id] = { + "roles": set(), + "paths": set(), + "fingerprints": {key: set() for key in FINGERPRINT_KINDS if key != "image_sha256"}, + } + group = groups[group_id] + for key, value in fingerprints.items(): + group["fingerprints"][key].add(value) + group["roles"].add(str(record["role"])) + group["paths"].add(str(record["image"]["relative_path"])) + + units = [] + blockers = [] + for group_id, group in groups.items(): + roles = group["roles"] + paths = group["paths"] + if len(roles) != 1 or len(paths) != 1 or any( + len(values) != 1 for values in group["fingerprints"].values() + ): + blockers.append({ + "code": "GROUP_INPUT_INCONSISTENT", + "pair_group_id": group_id, + "roles": sorted(roles), + "paths": sorted(paths), + "fingerprint_cardinality": { + key: len(values) for key, values in group["fingerprints"].items() + }, + }) + continue + path = image_root / next(iter(paths)) + if not path.is_file(): + blockers.append({"code": "IMAGE_MISSING", "path": str(path)}) + continue + fingerprints = { + key: next(iter(values)) for key, values in group["fingerprints"].items() + } + fingerprints["image_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + units.append({ + "pair_group_id": group_id, + "role": next(iter(roles)), + "path": str(path), + "dhash": _dhash(path), + **fingerprints, + }) + + exact_overlaps: dict[str, list[dict[str, Any]]] = {} + for kind in FINGERPRINT_KINDS: + by_value: dict[str, list[dict[str, Any]]] = defaultdict(list) + for unit in units: + by_value[str(unit[kind])].append(unit) + hits = [] + for value, values in by_value.items(): + roles = sorted({item["role"] for item in values}) + if len(roles) > 1: + hits.append({ + "fingerprint": value, + "roles": roles, + "pair_group_ids": sorted(item["pair_group_id"] for item in values), + }) + exact_overlaps[kind] = hits + if hits: + blockers.append({ + "code": "CROSS_ROLE_EXACT_OVERLAP", + "kind": kind, + "count": len(hits), + }) + + dhash_candidates = [] + perceptual_hits = [] + phash_cache: dict[str, int] = {} + phash_distance = 6 + for left, right in itertools.combinations(units, 2): + if left["role"] == right["role"]: + continue + distance = _hamming(int(left["dhash"]), int(right["dhash"])) + if distance <= perceptual_distance: + for unit in (left, right): + path = str(unit["path"]) + if path not in phash_cache: + phash_cache[path] = _phash(Path(path)) + phash_value = _hamming(phash_cache[str(left["path"])], phash_cache[str(right["path"])]) + candidate = { + "left_group": left["pair_group_id"], + "left_role": left["role"], + "right_group": right["pair_group_id"], + "right_role": right["role"], + "dhash_distance": distance, + "phash_distance": phash_value, + } + dhash_candidates.append(candidate) + if phash_value <= phash_distance: + perceptual_hits.append(candidate) + if perceptual_hits: + blockers.append({ + "code": "CROSS_ROLE_PERCEPTUAL_OVERLAP", + "count": len(perceptual_hits), + "threshold": { + "dhash": perceptual_distance, + "phash": phash_distance, + }, + }) + + return { + "schema_version": "mpr-tsr/raw-preserving-overlap-audit-v1", + "records": record_count, + "pair_groups": len(groups), + "audited_units": len(units), + "perceptual_hash": "dhash-9x8-screened-by-phash-32x32", + "perceptual_hamming_threshold": { + "dhash": perceptual_distance, + "phash": phash_distance, + }, + "exact_cross_role_overlaps": exact_overlaps, + "dhash_cross_role_candidate_count": len(dhash_candidates), + "dhash_cross_role_candidate_examples": dhash_candidates[:100], + "perceptual_cross_role_overlaps": perceptual_hits, + "blocker_count": len(blockers), + "blockers": blockers, + "status": "PASS" if not blockers else "BLOCKED", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fail-closed cross-role overlap audit.") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--image-root", type=Path, required=True) + parser.add_argument("--perceptual-distance", type=int, default=4) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = audit(_load(args.input), args.image_root, args.perceptual_distance) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({ + "status": report["status"], + "pair_groups": report["pair_groups"], + "audited_units": report["audited_units"], + "blocker_count": report["blocker_count"], + }, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_raw_preserving_split.py b/scripts/audit_raw_preserving_split.py new file mode 100644 index 0000000..050c2c6 --- /dev/null +++ b/scripts/audit_raw_preserving_split.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +import argparse +import collections +import json +from pathlib import Path +from typing import Any, Iterable, Iterator + + +ROLE_VALUES = ("train", "development", "holdout", "terminal") +FAMILY_KEYS = ( + "document_cluster_id", + "template_family_id", + "content_family_id", + "renderer_family_id", + "generation_seed", +) + + +def _load(path: Path) -> Iterator[dict[str, Any]]: + with path.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, 1): + if not line.strip(): + continue + value = json.loads(line) + if not isinstance(value, dict): + raise ValueError(f"line {line_number}: record is not an object") + yield value + + +def _family_value(record: dict[str, Any], key: str) -> Any: + provenance = record.get("provenance") + if not isinstance(provenance, dict): + return None + return provenance.get(key) + + +def audit(records: Iterable[dict[str, Any]], expected_roles: set[str]) -> dict[str, Any]: + blockers: list[dict[str, Any]] = [] + groups: dict[str, dict[str, Any]] = {} + sample_ids: set[str] = set() + duplicate_sample_ids = 0 + record_count = 0 + role_counts: collections.Counter[str] = collections.Counter() + missing_fields: collections.Counter[str] = collections.Counter() + family_roles: dict[str, dict[str, set[str]]] = { + key: collections.defaultdict(set) for key in FAMILY_KEYS + } + source_hash_by_family: dict[str, set[str]] = { + key: set() for key in ("template_family_id", "content_family_id") + } + + for index, record in enumerate(records): + record_count += 1 + sample_id = record.get("sample_id") + if not isinstance(sample_id, str) or not sample_id: + blockers.append({"code": "MISSING_SAMPLE_ID", "record_index": index}) + elif sample_id in sample_ids: + duplicate_sample_ids += 1 + else: + sample_ids.add(sample_id) + role = record.get("role") + if role not in ROLE_VALUES: + blockers.append({ + "code": "INVALID_ROLE", + "sample_id": sample_id, + "role": role, + }) + else: + role_counts[role] += 1 + group_id = record.get("pair_group_id") + if not isinstance(group_id, str) or not group_id: + blockers.append({"code": "MISSING_PAIR_GROUP_ID", "sample_id": sample_id}) + else: + group = groups.setdefault(group_id, { + "size": 0, + "roles": set(), + "families": {key: set() for key in FAMILY_KEYS}, + }) + group["size"] += 1 + group["roles"].add(role) + for key in FAMILY_KEYS: + group["families"][key].add(_family_value(record, key)) + source_hash = record.get("image", {}).get("source_document_hash") + for key in FAMILY_KEYS: + value = _family_value(record, key) + if value is None or value == "": + missing_fields[key] += 1 + elif role in ROLE_VALUES: + family_roles[key][str(value)].add(role) + if key in source_hash_by_family and value is not None and source_hash is not None: + if str(value) == str(source_hash): + source_hash_by_family[key].add(str(value)) + provenance = record.get("provenance") + if not isinstance(provenance, dict) or provenance.get("terminal_data_used") is not False: + blockers.append({"code": "TERMINAL_DATA_FLAG_NOT_FALSE", "sample_id": sample_id}) + + if duplicate_sample_ids: + blockers.append({"code": "DUPLICATE_SAMPLE_ID", "count": duplicate_sample_ids}) + + missing_roles = sorted(expected_roles - set(role_counts)) + if missing_roles: + blockers.append({"code": "EXPECTED_ROLE_MISSING", "roles": missing_roles}) + + for key, count in missing_fields.items(): + if count: + blockers.append({"code": "FAMILY_FIELD_MISSING", "field": key, "records": count}) + + for key, values in family_roles.items(): + crossed = {value: sorted(roles) for value, roles in values.items() if len(roles) > 1} + if crossed: + blockers.append({ + "code": "FAMILY_CROSSES_ROLES", + "field": key, + "values": crossed, + }) + if expected_roles and len(values) < len(expected_roles): + blockers.append({ + "code": "INSUFFICIENT_FAMILY_CARDINALITY", + "field": key, + "distinct_values": len(values), + "required_roles": len(expected_roles), + }) + + for key, values in source_hash_by_family.items(): + if values: + blockers.append({ + "code": "FAMILY_ALIASES_SOURCE_DOCUMENT", + "field": key, + "values": len(values), + }) + + group_sizes = collections.Counter(group["size"] for group in groups.values()) + inconsistent_groups = [] + for group_id, group in groups.items(): + if len(group["roles"]) != 1 or any( + len(snapshot) != 1 for snapshot in group["families"].values() + ): + inconsistent_groups.append(group_id) + if inconsistent_groups: + blockers.append({ + "code": "PAIR_GROUP_SPLIT_OR_INCONSISTENT", + "groups": inconsistent_groups, + }) + + return { + "schema_version": "mpr-tsr/raw-preserving-split-audit-v1", + "records": record_count, + "pair_groups": len(groups), + "role_counts": dict(role_counts), + "group_size_histogram": dict(group_sizes), + "missing_family_fields": dict(missing_fields), + "family_cardinality": { + key: len(values) for key, values in family_roles.items() + }, + "family_cross_role_values": { + key: { + value: sorted(roles) + for value, roles in values.items() + if len(roles) > 1 + } + for key, values in family_roles.items() + }, + "source_hash_alias_counts": { + key: len(values) for key, values in source_hash_by_family.items() + }, + "expected_roles": sorted(expected_roles), + "missing_expected_roles": missing_roles, + "blocker_count": len(blockers), + "blockers": blockers, + "status": "PASS" if not blockers else "BLOCKED", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Fail-closed Raw-preserving family split audit.") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--expected-role", + action="append", + choices=ROLE_VALUES, + default=["train", "development", "holdout"], + ) + args = parser.parse_args() + report = audit(_load(args.input), set(args.expected_role)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({ + "status": report["status"], + "records": report["records"], + "pair_groups": report["pair_groups"], + "blocker_count": report["blocker_count"], + }, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_selector_shortcuts.py b/scripts/audit_selector_shortcuts.py new file mode 100644 index 0000000..335c4a3 --- /dev/null +++ b/scripts/audit_selector_shortcuts.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import argparse +import collections +import json +from pathlib import Path +from typing import Any, Callable, Iterable, Iterator + + +NUMERIC_FEATURES = ( + "cell_count_delta", + "row_delta", + "col_delta", +) +CATEGORICAL_FEATURES = ( + "identity_candidate", + "text_multiset_equal", + "structure_equal", +) + + +def _load(path: Path) -> Iterator[dict[str, Any]]: + with path.open(encoding="utf-8") as handle: + for line in handle: + if line.strip(): + yield json.loads(line) + + +def _cells(record: dict[str, Any]) -> list[dict[str, Any]]: + return list(record.get("canonical_table", {}).get("cells", [])) + + +def _structure(cell: dict[str, Any]) -> tuple[Any, ...]: + return ( + cell.get("row", cell.get("row_start")), + cell.get("col", cell.get("col_start")), + cell.get("rowspan", cell.get("row_end")), + cell.get("colspan", cell.get("col_end")), + cell.get("tag", "td"), + ) + + +def _features(record: dict[str, Any]) -> dict[str, Any]: + raw = record["raw_record"] + candidate = record["candidate_record"] + raw_table = raw["canonical_table"] + candidate_table = candidate["canonical_table"] + raw_cells = _cells(raw) + candidate_cells = _cells(candidate) + return { + "cell_count_delta": len(candidate_cells) - len(raw_cells), + "row_delta": int(candidate_table.get("rows", 0)) - int(raw_table.get("rows", 0)), + "col_delta": int(candidate_table.get("cols", 0)) - int(raw_table.get("cols", 0)), + "identity_candidate": raw_table == candidate_table, + "text_multiset_equal": sorted(str(cell.get("text", "")) for cell in raw_cells) == sorted(str(cell.get("text", "")) for cell in candidate_cells), + "structure_equal": sorted(_structure(cell) for cell in raw_cells) == sorted(_structure(cell) for cell in candidate_cells), + } + + +def _metrics(labels: list[bool], predictions: list[bool]) -> dict[str, float | int]: + tp = sum(label and prediction for label, prediction in zip(labels, predictions)) + fp = sum(not label and prediction for label, prediction in zip(labels, predictions)) + tn = sum(not label and not prediction for label, prediction in zip(labels, predictions)) + fn = sum(label and not prediction for label, prediction in zip(labels, predictions)) + tpr = tp / (tp + fn) if tp + fn else 0.0 + tnr = tn / (tn + fp) if tn + fp else 0.0 + return { + "tp": tp, + "fp": fp, + "tn": tn, + "fn": fn, + "accuracy": (tp + tn) / len(labels) if labels else 0.0, + "balanced_accuracy": (tpr + tnr) / 2, + "precision": tp / (tp + fp) if tp + fp else 0.0, + "recall": tpr, + } + + +def audit(records: Iterable[dict[str, Any]]) -> dict[str, Any]: + labels = [] + values = [] + for record in records: + labels.append(record["oracle_decision"]["action"] == "ACCEPT_EDIT") + values.append(_features(record)) + rules = [] + + for feature in NUMERIC_FEATURES: + candidates = sorted({float(value[feature]) for value in values}) + thresholds = sorted(set(candidates + [(left + right) / 2 for left, right in zip(candidates, candidates[1:])])) + for threshold in thresholds: + for direction in ("greater", "less"): + if direction == "greater": + predictions = [float(value[feature]) > threshold for value in values] + else: + predictions = [float(value[feature]) < threshold for value in values] + rules.append({ + "feature": feature, + "rule": f"{direction}_than_{threshold:g}", + **_metrics(labels, predictions), + }) + + for feature in CATEGORICAL_FEATURES: + for positive_value in (True, False): + predictions = [bool(value[feature]) is positive_value for value in values] + rules.append({ + "feature": feature, + "rule": f"equals_{str(positive_value).lower()}", + **_metrics(labels, predictions), + }) + + rules.sort(key=lambda value: (float(value["balanced_accuracy"]), float(value["precision"]), float(value["recall"])), reverse=True) + all_keep = _metrics(labels, [False] * len(labels)) + best = rules[0] if rules else None + blocked = bool(best and best["balanced_accuracy"] >= 0.90) + contingency = { + feature: { + str(feature_value): dict(collections.Counter( + "ACCEPT_EDIT" if label else "KEEP_RAW" + for value, label in zip(values, labels) + if value[feature] == feature_value + )) + for feature_value in sorted({value[feature] for value in values}, key=str) + } + for feature in (*NUMERIC_FEATURES, *CATEGORICAL_FEATURES) + } + return { + "schema_version": "mpr-tsr/selector-shortcut-audit-v1", + "records": len(labels), + "keep_records": sum(not label for label in labels), + "accept_records": sum(labels), + "all_keep": all_keep, + "best_single_feature_rule": best, + "top_rules": rules[:10], + "feature_label_contingency": contingency, + "block_threshold_balanced_accuracy": 0.90, + "status": "BLOCKED_SHORTCUT" if blocked else "PASS", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Audit synthetic KEEP/EDIT pairs for observable shortcut labels.") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = audit(_load(args.input)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({ + "status": report["status"], + "records": report["records"], + "best_single_feature_rule": report["best_single_feature_rule"], + "all_keep": report["all_keep"], + }, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_raw_preserving_family_plan.py b/scripts/build_raw_preserving_family_plan.py new file mode 100644 index 0000000..f54e193 --- /dev/null +++ b/scripts/build_raw_preserving_family_plan.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +TEMPLATE_CATALOG = ( + {"role": "train", "tables": 1, "layout": "stacked", "border": "none", "merged": False, "header_depth": 1, "density": "medium", "orientation": "portrait", "columns": 4, "body_rows": [9, 12]}, + {"role": "train", "tables": 1, "layout": "stacked", "border": "three_line", "merged": True, "header_depth": 2, "density": "dense", "orientation": "portrait", "columns": 5, "body_rows": [16, 20]}, + {"role": "train", "tables": 1, "layout": "stacked", "border": "full", "merged": True, "header_depth": 2, "density": "sparse", "orientation": "landscape", "columns": 6, "body_rows": [5, 8]}, + {"role": "train", "tables": 2, "layout": "stacked", "border": "none", "merged": True, "header_depth": 1, "density": "medium", "orientation": "portrait", "columns": 5, "body_rows": [10, 14]}, + {"role": "train", "tables": 2, "layout": "side_by_side", "border": "three_line", "merged": False, "header_depth": 1, "density": "medium", "orientation": "landscape", "columns": 4, "body_rows": [9, 13]}, + {"role": "train", "tables": 3, "layout": "stacked", "border": "none", "merged": True, "header_depth": 2, "density": "dense", "orientation": "portrait", "columns": 6, "body_rows": [16, 21]}, + {"role": "train", "tables": 2, "layout": "stacked", "border": "full", "merged": False, "header_depth": 2, "density": "sparse", "orientation": "portrait", "columns": 5, "body_rows": [5, 8]}, + {"role": "train", "tables": 1, "layout": "stacked", "border": "none", "merged": True, "header_depth": 2, "density": "dense", "orientation": "landscape", "columns": 6, "body_rows": [18, 24]}, + {"role": "development", "tables": 1, "layout": "stacked", "border": "three_line", "merged": True, "header_depth": 1, "density": "medium", "orientation": "landscape", "columns": 7, "body_rows": [11, 15]}, + {"role": "development", "tables": 2, "layout": "side_by_side", "border": "none", "merged": True, "header_depth": 2, "density": "dense", "orientation": "landscape", "columns": 7, "body_rows": [16, 22]}, + {"role": "holdout", "tables": 3, "layout": "stacked", "border": "three_line", "merged": False, "header_depth": 1, "density": "medium", "orientation": "portrait", "columns": 8, "body_rows": [10, 14]}, + {"role": "holdout", "tables": 1, "layout": "stacked", "border": "full", "merged": False, "header_depth": 2, "density": "dense", "orientation": "portrait", "columns": 9, "body_rows": [17, 23]}, + {"role": "holdout", "tables": 2, "layout": "stacked", "border": "three_line", "merged": True, "header_depth": 2, "density": "medium", "orientation": "portrait", "columns": 8, "body_rows": [11, 16]}, + {"role": "holdout", "tables": 2, "layout": "side_by_side", "border": "full", "merged": True, "header_depth": 1, "density": "sparse", "orientation": "landscape", "columns": 9, "body_rows": [5, 9]}, +) + +RENDERER_CATALOG = ( + {"role": "train", "profile": "clean"}, + {"role": "train", "profile": "jpeg_light"}, + {"role": "train", "profile": "blur_light"}, + {"role": "development", "profile": "downsample_light"}, + {"role": "holdout", "profile": "low_contrast"}, + {"role": "holdout", "profile": "jpeg_medium"}, +) + + +def _role(index: int, groups: int, train_ratio: float, development_ratio: float) -> str: + train_end = int(groups * train_ratio) + development_end = train_end + int(groups * development_ratio) + if index < train_end: + return "train" + if index < development_end: + return "development" + return "holdout" + + +def _catalog_entries(catalog: tuple[dict, ...], role: str) -> list[tuple[int, dict]]: + return [(index, value) for index, value in enumerate(catalog) if value["role"] == role] + + +def build_plan(groups: int, seed: int, train_ratio: float, development_ratio: float) -> list[dict]: + if groups <= 0: + raise ValueError("groups must be positive") + if not 0 < train_ratio < 1 or not 0 < development_ratio < 1: + raise ValueError("split ratios must be between zero and one") + if train_ratio + development_ratio >= 1: + raise ValueError("train and development ratios must leave holdout groups") + + per_role_index = {"train": 0, "development": 0, "holdout": 0} + plan = [] + for index in range(groups): + role = _role(index, groups, train_ratio, development_ratio) + role_index = per_role_index[role] + per_role_index[role] += 1 + template_entries = _catalog_entries(TEMPLATE_CATALOG, role) + renderer_entries = _catalog_entries(RENDERER_CATALOG, role) + template_index, template_spec = template_entries[role_index % len(template_entries)] + renderer_index, renderer_spec = renderer_entries[role_index % len(renderer_entries)] + language = "en" if role_index % 4 == 0 else "cn" + generation_seed = seed * 1_000_003 + index + document_cluster_id = f"raw-preserving-v2-doc-{index:06d}" + template_family_id = f"raw-preserving-v2-template-{template_index:02d}" + content_family_id = f"raw-preserving-v2-content-{index:06d}" + renderer_family_id = f"raw-preserving-v2-renderer-{renderer_index:02d}" + identity = { + "group_index": index, + "role": role, + "document_cluster_id": document_cluster_id, + "template_family_id": template_family_id, + "content_family_id": content_family_id, + "renderer_family_id": renderer_family_id, + "generation_seed": generation_seed, + "language": language, + "template_spec": template_spec, + "renderer_spec": renderer_spec, + } + plan.append({ + "plan_version": "mpr-tsr/raw-preserving-family-plan-v2", + "group_index": index, + "pair_group_id": f"raw-preserving-v2-group-{index:06d}", + **identity, + "plan_entry_sha256": hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest(), + }) + return plan + + +def main() -> int: + parser = argparse.ArgumentParser(description="Build a terminal-blind pre-render family split plan.") + parser.add_argument("--groups", type=int, required=True) + parser.add_argument("--seed", type=int, default=20260813) + parser.add_argument("--train-ratio", type=float, default=0.70) + parser.add_argument("--development-ratio", type=float, default=0.10) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + plan = build_plan(args.groups, args.seed, args.train_ratio, args.development_ratio) + args.out.parent.mkdir(parents=True, exist_ok=True) + with args.out.open("w", encoding="utf-8") as handle: + for entry in plan: + handle.write(json.dumps(entry, ensure_ascii=False) + "\n") + role_counts = {} + for entry in plan: + role_counts[entry["role"]] = role_counts.get(entry["role"], 0) + 1 + print(json.dumps({ + "groups": len(plan), + "role_counts": role_counts, + "template_families": len({entry["template_family_id"] for entry in plan}), + "renderer_families": len({entry["renderer_family_id"] for entry in plan}), + }, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_synthfin_raw_preserving_smoke.py b/scripts/build_synthfin_raw_preserving_smoke.py new file mode 100644 index 0000000..f689452 --- /dev/null +++ b/scripts/build_synthfin_raw_preserving_smoke.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import random +from pathlib import Path +from typing import Any + +from mpr_tsr_splitmerge_v2.counterfactual import build_candidate +from mpr_tsr_splitmerge_v2.raw_preserving import KEEP, label_oracle_action + + +def _provenance(sample_id: str, source_hash: str, purpose: str) -> dict[str, Any]: + return { + "sample_id": sample_id, + "producer": "raw-preserving-synthfin-adapter", + "producer_version": "v1", + "purpose": purpose, + "input_image_sha256": source_hash, + "terminal_benchmarks_visible": False, + } + + +def _cell_token_count(text: str) -> int: + return max(1, len(text)) if text else 0 + + +def _canonical_table(table: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + cells = [] + tokens = [] + for index, source in enumerate(table.get("cells", [])): + row = int(source["row_start"]) + col = int(source["col_start"]) + row_end = int(source["row_end"]) + col_end = int(source["col_end"]) + text = str(source.get("text", "") or "") + bbox = [float(value) for value in source["bbox"]] + token_indexes = list(range(len(tokens), len(tokens) + _cell_token_count(text))) + if text: + token_width = (bbox[2] - bbox[0]) / len(text) + for char_index, char in enumerate(text): + token_bbox = [ + bbox[0] + char_index * token_width, + bbox[1], + bbox[0] + (char_index + 1) * token_width, + bbox[3], + ] + tokens.append({"text": char, "bbox": token_bbox}) + cells.append({ + "cell_id": f"gold-{index:05d}", + "row": row, + "col": col, + "rowspan": row_end - row + 1, + "colspan": col_end - col + 1, + "text": text, + "tag": "th" if source.get("is_header") else "td", + "bbox": bbox, + "geometry": {"status": "present", "bbox": bbox}, + "ocr_token_indexes": token_indexes, + }) + return { + "rows": int(table["n_rows"]), + "cols": int(table["n_cols"]), + "cells": cells, + }, tokens + + +def _record( + table: dict[str, Any], + *, + sample_id: str, + source_hash: str, + purpose: str, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + canonical, tokens = _canonical_table(table) + return { + "canonical_table": canonical, + "ocr_tokens": tokens, + "non_table_state_sha256": hashlib.sha256( + f"{source_hash}:non-table".encode("utf-8") + ).hexdigest(), + "provenance": _provenance(sample_id, source_hash, purpose), + }, tokens + + +def _clone_with_provenance(record: dict[str, Any], purpose: str) -> dict[str, Any]: + value = copy.deepcopy(record) + value["provenance"]["purpose"] = purpose + return value + + +def _load_tables(root: Path, limit: int, one_table_per_page: bool = False) -> list[dict[str, Any]]: + entries: list[dict[str, Any]] = [] + manifests = sorted(root.glob("shard_*/manifest.jsonl")) + root_manifest = root / "manifest.jsonl" + if root_manifest.is_file(): + manifests = [root_manifest, *manifests] + for manifest in manifests: + with manifest.open(encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + page = json.loads(line) + for table in page.get("tables") or []: + if not table.get("cells") or not table.get("n_rows") or not table.get("n_cols"): + continue + entries.append({"page": page, "table": table, "manifest": str(manifest)}) + if len(entries) >= limit: + return entries + if one_table_per_page: + break + return entries + + +def _validate_generator_policy(page: dict[str, Any]) -> None: + if page.get("terminal_data_used") is not False: + raise ValueError("formal corpus requires terminal_data_used=false") + if page.get("generator_policy") != "terminal_blind_v1": + raise ValueError("formal corpus requires generator_policy=terminal_blind_v1") + identity = " ".join( + str(page.get(key, "")) + for key in ("renderer_version", "generator_policy", "source_dataset") + ).lower() + forbidden = ("eval50", "eval_50", "omnidoc", "customer50", "customer_50") + matches = [token for token in forbidden if token in identity] + if matches: + raise ValueError(f"formal corpus generator identity is terminal-derived: {matches}") + + +def _family_fields(page: dict[str, Any], require_family_plan: bool) -> dict[str, Any]: + if require_family_plan: + _validate_generator_policy(page) + keys = ( + "role", + "pair_group_id", + "document_cluster_id", + "template_family_id", + "content_family_id", + "renderer_family_id", + "generation_seed", + "family_plan_entry_sha256", + ) + missing = [key for key in keys if page.get(key) in (None, "")] + if require_family_plan and missing: + raise ValueError(f"manifest page lacks family-plan fields: {missing}") + source_hash = str(page["source_document_hash"]) + legacy_seed = int(source_hash[:16], 16) + return { + "role": page.get("role", "development"), + "pair_group_id": page.get("pair_group_id"), + "document_cluster_id": page.get("document_cluster_id", f"source-{source_hash}"), + "template_family_id": page.get("template_family_id", source_hash), + "content_family_id": page.get("content_family_id", source_hash), + "renderer_family_id": page.get( + "renderer_family_id", page.get("renderer_version", "unknown") + ), + "generation_seed": page.get("generation_seed", legacy_seed), + "family_plan_entry_sha256": page.get("family_plan_entry_sha256"), + } + + +def _make_pair( + entry: dict[str, Any], + group_index: int, + local_index: int, + *, + dataset_root: Path, + raw: dict[str, Any], + gold: dict[str, Any], + candidate: dict[str, Any] | None, + ocr_tokens: list[dict[str, Any]], + operator: str, + tag: str, + require_family_plan: bool, +) -> dict[str, Any]: + page = entry["page"] + source_hash = str(page["source_document_hash"]) + family = _family_fields(page, require_family_plan) + table_index = int(entry["table"]["table_index"]) + pair_group_id = family["pair_group_id"] + if pair_group_id: + pair_group_id = f"{pair_group_id}-table-{table_index:02d}" + else: + pair_group_id = f"synthfin-v34-group-{group_index:04d}" + sample_id = ( + f"{pair_group_id}-cf-{local_index:02d}" + if require_family_plan + else f"synthfin-v34-rp-{group_index:04d}-{local_index:02d}" + ) + decision = label_oracle_action(raw, candidate, gold, ocr_tokens=ocr_tokens) + image = Path(str(page["image"])) + image_path = image if image.is_absolute() else dataset_root / image + if not image_path.is_file(): + raise ValueError(f"image does not exist: {image_path}") + if not image.is_absolute(): + relative_image = image.as_posix() + else: + relative_image = os.path.relpath(str(image), str(dataset_root)).replace(os.sep, "/") + if require_family_plan and (Path(relative_image).is_absolute() or ".." in Path(relative_image).parts): + raise ValueError("formal image path must be relative to the dataset root") + image_sha256 = hashlib.sha256(image_path.read_bytes()).hexdigest() + declared_image_sha256 = page.get("image_sha256") + if require_family_plan and declared_image_sha256 != image_sha256: + raise ValueError("formal image SHA256 does not match the manifest") + return { + "schema_version": "mpr-tsr/raw-preserving-pair-v1", + "sample_id": sample_id, + "pair_group_id": pair_group_id, + "role": family["role"], + "image": { + "relative_path": relative_image, + "sha256": image_sha256, + "source_document_hash": source_hash, + "table_index": table_index, + "image_size": page.get("image_size"), + }, + "raw_record": raw, + "gold_record": gold, + "candidate_record": candidate, + "oracle_decision": decision.as_dict(), + "phenomenon_tags": [tag, str(entry["table"].get("meta", {}).get("border", "unknown"))], + "operator": {"name": operator, "version": "v1", "terminal_data_used": False}, + "provenance": { + "generator": str(page.get("source_dataset", "unknown")), + "generator_version": str(page.get("renderer_version", "unknown")), + **( + {"generator_policy": str(page["generator_policy"])} + if page.get("generator_policy") is not None else {} + ), + "generation_seed": family["generation_seed"], + "terminal_data_used": False, + "document_cluster_id": family["document_cluster_id"], + "template_family_id": family["template_family_id"], + "content_family_id": family["content_family_id"], + "renderer_family_id": family["renderer_family_id"], + **( + {"family_plan_entry_sha256": str(family["family_plan_entry_sha256"])} + if family["family_plan_entry_sha256"] is not None else {} + ), + "source_dataset": str(page.get("source_dataset", "SynthFin")), + "license": str(page.get("license", "synthetic")), + **({"font": page["font"]} if page.get("font") is not None else {}), + }, + } + + +def build_group( + entry: dict[str, Any], + group_index: int, + *, + dataset_root: Path, + require_family_plan: bool, +) -> list[dict[str, Any]]: + page = entry["page"] + table = entry["table"] + source_hash = str(page["source_document_hash"]) + gold, ocr_tokens = _record(table, sample_id=f"gold-{group_index:04d}", source_hash=source_hash, purpose="offline_gold") + raw_good = _clone_with_provenance(gold, "raw_like_identity") + harmful = build_candidate(raw_good, gold, operator="over_merge") + if harmful is None: + harmful = build_candidate(raw_good, gold, operator="over_split") + if harmful is None: + harmful = _clone_with_provenance(raw_good, "raw_like_identity_fallback") + raw_bad = build_candidate( + gold, + gold, + operator="assignment_swap" if require_family_plan else "over_merge", + ) + if raw_bad is None: + raw_bad = build_candidate(gold, gold, operator="over_merge") + if raw_bad is None: + raw_bad = copy.deepcopy(gold) + cells = raw_bad["canonical_table"]["cells"] + if cells: + cells[0]["text"] = f"{cells[0]['text']}__raw_error" + records = [] + specifications = [ + (raw_good, _clone_with_provenance(raw_good, "identity_candidate"), "identity", "raw_good_identity"), + (raw_good, harmful, "over_edit", "raw_good_harmful_candidate"), + (raw_good, harmful, "over_edit_repeat", "raw_good_harmful_candidate_repeat"), + (raw_good, _clone_with_provenance(raw_good, "identity_candidate_repeat"), "identity_repeat", "raw_good_identity_repeat"), + (raw_bad, _clone_with_provenance(raw_bad, "identity_candidate"), "identity", "raw_bad_identity"), + (raw_bad, harmful, "over_edit", "raw_bad_harmful_candidate"), + (raw_bad, _clone_with_provenance(raw_bad, "identity_candidate_tie"), "identity_tie", "raw_bad_tie"), + (raw_bad, harmful, "partial_candidate", "raw_bad_partial_or_harmful"), + (raw_bad, _clone_with_provenance(raw_bad, "non_table_change_candidate"), "non_table_change", "raw_bad_non_table_control"), + (raw_bad, _clone_with_provenance(gold, "gold_candidate"), "gold_candidate", "raw_bad_gold_candidate"), + ] + for index, (raw_value, candidate, operator, tag) in enumerate(specifications): + if operator == "non_table_change": + candidate = copy.deepcopy(candidate) + candidate["non_table_state_sha256"] = "changed-non-table-state" + records.append(_make_pair( + entry, group_index, index, dataset_root=dataset_root, raw=raw_value, gold=gold, + candidate=candidate, ocr_tokens=ocr_tokens, operator=operator, tag=tag, + require_family_plan=require_family_plan, + )) + return records + + +def main() -> int: + parser = argparse.ArgumentParser(description="Build a 256-group SynthFin Raw-preserving smoke manifest.") + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--groups", type=int, default=256) + parser.add_argument("--require-family-plan", action="store_true") + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + entries = _load_tables( + args.root, + args.groups, + one_table_per_page=args.require_family_plan, + ) + if len(entries) != args.groups: + raise RuntimeError(f"requested {args.groups} table groups, found {len(entries)}") + records = [ + record + for index, entry in enumerate(entries) + for record in build_group( + entry, + index, + dataset_root=args.root, + require_family_plan=args.require_family_plan, + ) + ] + counts: dict[str, int] = {} + for record in records: + action = record["oracle_decision"]["action"] + counts[action] = counts.get(action, 0) + 1 + expected = {KEEP: args.groups * 9, "ACCEPT_EDIT": args.groups} + if counts != expected: + raise RuntimeError(f"oracle distribution mismatch: expected={expected} observed={counts}") + args.out.parent.mkdir(parents=True, exist_ok=True) + with args.out.open("w", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + print(json.dumps({"groups": args.groups, "records": len(records), "action_counts": counts}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/render_terminal_blind_fin_tables.py b/scripts/render_terminal_blind_fin_tables.py new file mode 100644 index 0000000..10b4475 --- /dev/null +++ b/scripts/render_terminal_blind_fin_tables.py @@ -0,0 +1,327 @@ +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import random +from pathlib import Path +from typing import Any + +from PIL import Image, ImageDraw, ImageEnhance, ImageFilter, ImageFont + + +CN_ITEMS = ["营业收入", "营业成本", "流动资产", "固定资产", "应收账款", "存货", "短期借款", "所有者权益", "净利润", "现金及等价物"] +EN_ITEMS = ["Revenue", "Cost of sales", "Current assets", "Fixed assets", "Receivables", "Inventories", "Borrowings", "Total equity", "Net profit", "Cash equivalents"] +CN_TITLES = ["财务状况表", "经营成果表", "现金流量表", "主要财务指标"] +EN_TITLES = ["Financial Position", "Operating Results", "Cash Flows", "Key Financial Indicators"] +FONT_CANDIDATES = ( + Path("C:/Windows/Fonts/msyh.ttc"), + Path("C:/Windows/Fonts/simsun.ttc"), + Path("/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc"), + Path("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"), +) + + +def _font(path: Path | None, size: int) -> tuple[ImageFont.FreeTypeFont, dict[str, Any]]: + candidates = [path] if path is not None else list(FONT_CANDIDATES) + for candidate in candidates: + if candidate is not None and candidate.is_file(): + payload = candidate.read_bytes() + return ImageFont.truetype(str(candidate), size), { + "source": "system_or_user_provided", + "file_name": candidate.name, + "sha256": hashlib.sha256(payload).hexdigest(), + "redistributed": False, + } + raise RuntimeError("no usable font found; pass --font") + + +def _money(rng: random.Random) -> str: + value = rng.randint(1, 9_999_999) + text = f"{value:,}.{rng.randint(0, 99):02d}" + return f"({text})" if rng.random() < 0.12 else text + + +def _build_cells(rng: random.Random, spec: dict[str, Any], language: str) -> tuple[list[dict[str, Any]], int, int]: + density = str(spec["density"]) + n_body = rng.randint(int(spec["body_rows"][0]), int(spec["body_rows"][1])) + n_cols = int(spec["columns"]) + header_depth = int(spec["header_depth"]) + merged = bool(spec["merged"]) + items = CN_ITEMS if language == "cn" else EN_ITEMS + item_head = "项目" if language == "cn" else "Item" + periods = ("本期", "上期") if language == "cn" else ("Current", "Prior") + cells: list[dict[str, Any]] = [] + + if header_depth == 2: + cells.append({"r0": 0, "r1": 1, "c0": 0, "c1": 0, "text": item_head, "is_header": True}) + split = 1 + (n_cols - 1) // 2 + cells.append({"r0": 0, "r1": 0, "c0": 1, "c1": split - 1, "text": periods[0], "is_header": True}) + cells.append({"r0": 0, "r1": 0, "c0": split, "c1": n_cols - 1, "text": periods[1], "is_header": True}) + for col in range(1, n_cols): + cells.append({"r0": 1, "r1": 1, "c0": col, "c1": col, "text": "金额" if language == "cn" else "Amount", "is_header": True}) + else: + cells.append({"r0": 0, "r1": 0, "c0": 0, "c1": 0, "text": item_head, "is_header": True}) + for col in range(1, n_cols): + cells.append({"r0": 0, "r1": 0, "c0": col, "c1": col, "text": periods[(col - 1) % 2], "is_header": True}) + + row = header_depth + item_index = 0 + while row < header_depth + n_body: + remaining = header_depth + n_body - row + if merged and remaining >= 3 and item_index % 6 == 3: + text = "其中：核心业务" if language == "cn" else "Of which: core business" + cells.append({"r0": row, "r1": row, "c0": 0, "c1": n_cols - 1, "text": text, "is_header": False}) + row += 1 + continue + if merged and remaining >= 2 and item_index % 7 == 4: + cells.append({"r0": row, "r1": row + 1, "c0": 0, "c1": 0, "text": items[item_index % len(items)], "is_header": False}) + for offset in range(2): + for col in range(1, n_cols): + cells.append({"r0": row + offset, "r1": row + offset, "c0": col, "c1": col, "text": _money(rng), "is_header": False}) + row += 2 + item_index += 2 + continue + cells.append({"r0": row, "r1": row, "c0": 0, "c1": 0, "text": items[item_index % len(items)], "is_header": False}) + for col in range(1, n_cols): + cells.append({"r0": row, "r1": row, "c0": col, "c1": col, "text": "" if rng.random() < 0.05 else _money(rng), "is_header": False}) + row += 1 + item_index += 1 + return cells, row, n_cols + + +def _html(cells: list[dict[str, Any]], rows: int, cols: int) -> str: + origins = {(cell["r0"], cell["c0"]): cell for cell in cells} + owner = {} + for cell in cells: + for row in range(cell["r0"], cell["r1"] + 1): + for col in range(cell["c0"], cell["c1"] + 1): + if (row, col) in owner: + raise ValueError("overlapping cells") + owner[(row, col)] = (cell["r0"], cell["c0"]) + if len(owner) != rows * cols: + raise ValueError("incomplete grid") + output = [] + for row in range(rows): + nodes = [] + for col in range(cols): + if owner[(row, col)] != (row, col): + continue + cell = origins[(row, col)] + rowspan = cell["r1"] - cell["r0"] + 1 + colspan = cell["c1"] - cell["c0"] + 1 + attrs = (f' rowspan="{rowspan}"' if rowspan > 1 else "") + (f' colspan="{colspan}"' if colspan > 1 else "") + tag = "th" if cell["is_header"] else "td" + text = cell["text"].replace("&", "&").replace("<", "<").replace(">", ">") + nodes.append(f"<{tag}{attrs}>{text}") + output.append("" + "".join(nodes) + "") + return "" + "".join(output) + "
" + + +def _otsl(cells: list[dict[str, Any]], rows: int, cols: int) -> str: + origins = {(cell["r0"], cell["c0"]): cell for cell in cells} + owner = {} + for cell in cells: + for row in range(cell["r0"], cell["r1"] + 1): + for col in range(cell["c0"], cell["c1"] + 1): + owner[(row, col)] = (cell["r0"], cell["c0"]) + lines = [] + for row in range(rows): + tokens = [] + for col in range(cols): + origin = owner[(row, col)] + if origin == (row, col): + text = str(origins[origin]["text"]) + tokens.append(("" if not text else "") + text) + elif origin[0] == row: + tokens.append("") + elif origin[1] == col: + tokens.append("") + else: + tokens.append("") + lines.append("".join(tokens) + "") + return "\n".join(lines) + + +def _render_table(rng: random.Random, spec: dict[str, Any], language: str, width: int, font_path: Path | None) -> tuple[Image.Image, dict[str, Any], dict[str, Any]]: + density = str(spec["density"]) + font_size = {"sparse": 23, "medium": 19, "dense": 15}[density] + font, font_info = _font(font_path, font_size) + title_font, _ = _font(font_path, font_size + 5) + cells, rows, cols = _build_cells(rng, spec, language) + title = rng.choice(CN_TITLES if language == "cn" else EN_TITLES) + unit = "单位：人民币元" if language == "cn" else "Unit: currency" + top = font_size * 4 + row_h = font_size + {"sparse": 18, "medium": 12, "dense": 8}[density] + col_w = max(60, width // cols) + width = col_w * cols + image = Image.new("RGB", (width, top + rows * row_h + 2), "white") + draw = ImageDraw.Draw(image) + draw.text((4, 2), title, fill="black", font=title_font) + draw.text((width - 4, font_size + 10), unit, fill="black", font=font, anchor="ra") + grid_top = top + for cell in cells: + x0 = cell["c0"] * col_w + y0 = grid_top + cell["r0"] * row_h + x1 = (cell["c1"] + 1) * col_w + y1 = grid_top + (cell["r1"] + 1) * row_h + cell["bbox"] = [float(x0), float(y0), float(x1), float(y1)] + text = str(cell["text"]) + if text: + draw.text((x0 + 4, y0 + 3), text, fill="black", font=font) + if spec["border"] == "full": + draw.rectangle((x0, y0, x1, y1), outline="black", width=1) + if spec["border"] == "three_line": + draw.line((0, grid_top, width, grid_top), fill="black", width=2) + draw.line((0, grid_top + int(spec["header_depth"]) * row_h, width, grid_top + int(spec["header_depth"]) * row_h), fill="black", width=1) + draw.line((0, grid_top + rows * row_h, width, grid_top + rows * row_h), fill="black", width=2) + table = { + "cells": cells, + "html": _html(cells, rows, cols), + "otsl": _otsl(cells, rows, cols), + "n_rows": rows, + "n_cols": cols, + "meta": { + "header_depth": int(spec["header_depth"]), + "title": title, + "unit": unit, + "border": str(spec["border"]), + "lang": language, + }, + } + return image, table, font_info + + +def _apply_profile(image: Image.Image, profile: str) -> tuple[Image.Image, list[str], int]: + quality = 95 + operations = [] + if profile == "blur_light": + image = image.filter(ImageFilter.GaussianBlur(0.45)) + operations.append("gaussian_blur_0.45") + elif profile == "downsample_light": + original = image.size + image = image.resize((max(1, int(original[0] * 0.82)), max(1, int(original[1] * 0.82))), Image.Resampling.BICUBIC).resize(original, Image.Resampling.BICUBIC) + operations.append("downsample_0.82") + elif profile == "low_contrast": + image = ImageEnhance.Contrast(image).enhance(0.82) + operations.append("contrast_0.82") + elif profile == "jpeg_light": + quality = 88 + operations.append("jpeg_88") + elif profile == "jpeg_medium": + quality = 76 + operations.append("jpeg_76") + elif profile != "clean": + raise ValueError(f"unknown renderer profile: {profile}") + return image, operations, quality + + +def _page(entry: dict[str, Any], font_path: Path | None) -> tuple[Image.Image, list[dict[str, Any]], dict[str, Any], list[str], int]: + rng = random.Random(int(entry["generation_seed"])) + spec = dict(entry["template_spec"]) + language = str(entry["language"]) + portrait = spec["orientation"] == "portrait" + page_w, page_h = ((1240, 1754) if portrait else (1754, 1240)) + margin, gap = 70, 36 + table_count = int(spec["tables"]) + side = spec["layout"] == "side_by_side" + table_width = (page_w - 2 * margin - gap * (table_count - 1)) // table_count if side else page_w - 2 * margin + rendered = [_render_table(rng, spec, language, table_width, font_path) for _ in range(table_count)] + required_height = ( + 75 + max(image.height for image, _, _ in rendered) + margin + if side + else 75 + sum(image.height for image, _, _ in rendered) + gap * (table_count - 1) + margin + ) + page_h = max(page_h, required_height) + page = Image.new("RGB", (page_w, page_h), "white") + draw = ImageDraw.Draw(page) + body_font, font_info = _font(font_path, 18) + heading = "示例财务报告" if language == "cn" else "Illustrative Financial Report" + draw.text((margin, 24), heading, fill="black", font=body_font) + tables = [] + x, y = margin, 75 + for index, (image, table, _) in enumerate(rendered): + page.paste(image, (x, y)) + moved_cells = [] + for cell in table["cells"]: + value = dict(cell) + box = cell["bbox"] + value["bbox"] = [box[0] + x, box[1] + y, box[2] + x, box[3] + y] + value["row_start"] = int(value["r0"]) + value["row_end"] = int(value["r1"]) + value["col_start"] = int(value["c0"]) + value["col_end"] = int(value["c1"]) + moved_cells.append(value) + table_bbox = [float(x), float(y), float(x + image.width), float(y + image.height)] + tables.append({**table, "table_index": index, "bbox": table_bbox, "cells": moved_cells, "structure_sha256": hashlib.sha256(json.dumps([(c["r0"], c["r1"], c["c0"], c["c1"]) for c in moved_cells], separators=(",", ":")).encode()).hexdigest()}) + if side: + x += image.width + gap + else: + y += image.height + gap + profile = str(entry["renderer_spec"]["profile"]) + page, operations, quality = _apply_profile(page, profile) + return page, tables, font_info, operations, quality + + +def main() -> int: + parser = argparse.ArgumentParser(description="Render terminal-blind synthetic financial tables from a frozen family plan.") + parser.add_argument("--family-plan", type=Path, required=True) + parser.add_argument("--out-dir", type=Path, required=True) + parser.add_argument("--font", type=Path) + args = parser.parse_args() + if args.out_dir.exists(): + raise FileExistsError(f"output already exists: {args.out_dir}") + args.out_dir.mkdir(parents=True) + image_dir = args.out_dir / "images" + image_dir.mkdir() + entries = [json.loads(line) for line in args.family_plan.open(encoding="utf-8") if line.strip()] + manifest = args.out_dir / "manifest.jsonl" + with manifest.open("w", encoding="utf-8") as handle: + for entry in entries: + page, tables, font_info, operations, quality = _page(entry, args.font) + image_name = f"{entry['document_cluster_id']}.jpg" + image_path = image_dir / image_name + buffer = io.BytesIO() + page.save(buffer, format="JPEG", quality=quality) + payload = buffer.getvalue() + image_path.write_bytes(payload) + source_id = str(entry["document_cluster_id"]) + record = { + "schema_version": "terminal_blind_fin_page_v1", + "task": "table_page_and_crop", + "source_dataset": "TerminalBlindFinSynthetic", + "source_document_id": source_id, + "source_document_hash": hashlib.sha256(source_id.encode()).hexdigest(), + "source_document_hash_kind": "synthetic_id_sha256", + "license": "self-generated-code-output", + "deployment_status": "research_only_pending_license_review", + "renderer_version": "terminal_blind_fin_renderer_v1", + "generator_policy": "terminal_blind_v1", + "terminal_data_used": False, + "role": entry["role"], + "pair_group_id": entry["pair_group_id"], + "document_cluster_id": entry["document_cluster_id"], + "template_family_id": entry["template_family_id"], + "content_family_id": entry["content_family_id"], + "renderer_family_id": entry["renderer_family_id"], + "generation_seed": entry["generation_seed"], + "family_plan_entry_sha256": entry["plan_entry_sha256"], + "template_spec": entry["template_spec"], + "renderer_spec": entry["renderer_spec"], + "image": f"images/{image_name}", + "image_sha256": hashlib.sha256(payload).hexdigest(), + "image_size": [page.width, page.height], + "font": font_info, + "degradations": operations, + "language": entry["language"], + "n_tables": len(tables), + "tables": tables, + } + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + print(json.dumps({"accepted_pages": len(entries), "rejected": 0, "manifest": str(manifest)}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_k0_all_keep_audit.py b/scripts/run_k0_all_keep_audit.py new file mode 100644 index 0000000..5b9ed56 --- /dev/null +++ b/scripts/run_k0_all_keep_audit.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import argparse +import collections +import json +from pathlib import Path +from typing import Any + +from mpr_tsr_splitmerge_v2.safety_layer import ( + SafetyPolicy, + select_candidate_or_rollback, +) + + +def _load(path: Path) -> list[dict[str, Any]]: + with path.open(encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def run(records: list[dict[str, Any]]) -> dict[str, Any]: + policy = SafetyPolicy( + policy_id="k0-all-keep-v1", + minimum_expected_gain=0.0, + threshold_source="NONTERMINAL_PREREGISTERED", + text_policy="OCR_GROUNDED", + ) + decision_counts: collections.Counter[str] = collections.Counter() + reason_counts: collections.Counter[str] = collections.Counter() + failures = [] + + for record in records: + raw = record["raw_record"] + candidate = record.get("candidate_record") + ocr_tokens = raw.get("ocr_tokens") + selection = select_candidate_or_rollback( + raw, + candidate, + policy=policy, + expected_gain=None, + ocr_tokens=ocr_tokens, + ) + decision_counts[selection["decision"]] += 1 + reason_counts.update(selection["reason_codes"]) + checks = { + "not_accept": selection["decision"] != "ACCEPT", + "rollback_exact": selection["rollback_exact"] is True, + "state_hash_equal": ( + selection["output_state_sha256"] == selection["raw_state_sha256"] + ), + "non_table_state_equal": ( + selection["output"].get("non_table_state_sha256") + == raw.get("non_table_state_sha256") + ), + "output_equal_raw": selection["output"] == raw, + } + if not all(checks.values()): + failures.append({ + "sample_id": record.get("sample_id"), + "decision": selection["decision"], + "reason_codes": selection["reason_codes"], + "checks": checks, + }) + + return { + "schema_version": "mpr-tsr/k0-all-keep-audit-v1", + "records": len(records), + "decision_counts": dict(decision_counts), + "reason_counts": dict(reason_counts), + "exact_raw_outputs": len(records) - len(failures), + "failure_count": len(failures), + "failures": failures, + "status": "PASS" if not failures and decision_counts.get("ACCEPT", 0) == 0 else "FAIL", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run the K0 all-KEEP exact Raw bypass audit.") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = run(_load(args.input)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({ + "status": report["status"], + "records": report["records"], + "exact_raw_outputs": report["exact_raw_outputs"], + "decision_counts": report["decision_counts"], + "failure_count": report["failure_count"], + }, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_k1_geometry_selector.py b/scripts/run_k1_geometry_selector.py new file mode 100644 index 0000000..56c118a --- /dev/null +++ b/scripts/run_k1_geometry_selector.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import argparse +import collections +import json +import math +from pathlib import Path +from typing import Any, Iterable, Iterator + +from mpr_tsr_splitmerge_v2.safety_layer import ( + ExpectedGainEvidence, + SafetyPolicy, + select_candidate_or_rollback, + stable_sha256, + validate_candidate, +) +from mpr_tsr_splitmerge_v2.token_geometry import ownership_violations, topology_signature + + +def _load(path: Path, role: str) -> Iterator[dict[str, Any]]: + with path.open(encoding="utf-8") as handle: + for line in handle: + if line.strip(): + record = json.loads(line) + if record["role"] == role: + yield record + + +def _upper_zero_failures(trials: int, alpha: float = 0.05) -> float: + return 1.0 - alpha ** (1.0 / trials) if trials else 1.0 + + +def _lower_all_successes(successes: int, alpha: float = 0.05) -> float: + return alpha ** (1.0 / successes) if successes else 0.0 + + +def _token_indexes(record: dict[str, Any]) -> list[int] | None: + table = record.get("canonical_table", record) + values = [] + for cell in table.get("cells", []): + indexes = cell.get("ocr_token_indexes") + if not isinstance(indexes, list): + return None + values.extend(indexes) + return sorted(values) + + +def decide( + raw: dict[str, Any], + candidate: dict[str, Any], + ocr_tokens: list[dict[str, Any]], + policy: SafetyPolicy, +) -> tuple[bool, dict[str, Any]]: + raw_violations = ownership_violations(raw, ocr_tokens) + candidate_violations = ownership_violations(candidate, ocr_tokens) + topology_equal = topology_signature(raw) == topology_signature(candidate) + token_conserved = _token_indexes(raw) == _token_indexes(candidate) + non_table_equal = raw.get("non_table_state_sha256") == candidate.get("non_table_state_sha256") + validation = validate_candidate(raw, candidate, policy=policy, ocr_tokens=ocr_tokens) + accept = ( + raw_violations is not None + and candidate_violations is not None + and topology_equal + and token_conserved + and candidate_violations < raw_violations + and candidate_violations == 0 + and non_table_equal + and validation["status"] == "PASS" + ) + return accept, { + "raw_violations": raw_violations, + "candidate_violations": candidate_violations, + "topology_equal": topology_equal, + "token_conserved": token_conserved, + "non_table_equal": non_table_equal, + "validator_status": validation["status"], + } + + +def evaluate(records: Iterable[dict[str, Any]], role: str) -> dict[str, Any]: + selected = records + policy = SafetyPolicy( + policy_id="k1-geometry-selector-v1", + minimum_expected_gain=0.0, + threshold_source="NONTERMINAL_PREREGISTERED", + text_policy="OCR_GROUNDED", + ) + route_counts: collections.Counter[str] = collections.Counter() + final_counts: collections.Counter[str] = collections.Counter() + reason_counts: collections.Counter[str] = collections.Counter() + per_template: dict[str, collections.Counter[str]] = collections.defaultdict(collections.Counter) + group_routes: dict[str, collections.Counter[str]] = collections.defaultdict(collections.Counter) + failures = [] + + selected_count = 0 + for record in selected: + selected_count += 1 + raw = record["raw_record"] + candidate = record["candidate_record"] + tokens = raw.get("ocr_tokens", []) + predicted_edit, features = decide(raw, candidate, tokens, policy) + true_edit = record["oracle_decision"]["action"] == "ACCEPT_EDIT" + route_key = ( + "tp" if predicted_edit and true_edit else + "fp" if predicted_edit else + "fn" if true_edit else + "tn" + ) + route_counts[route_key] += 1 + group_routes[str(record["pair_group_id"])][route_key] += 1 + template = str(record["provenance"]["template_family_id"]) + per_template[template][route_key] += 1 + evidence = ( + ExpectedGainEvidence( + value=float(features["raw_violations"] - features["candidate_violations"]), + protocol_id="k1-geometry-selector-v1", + source="NONTERMINAL_DEVELOPMENT", + ) + if predicted_edit else None + ) + selection = select_candidate_or_rollback( + raw, + candidate, + policy=policy, + expected_gain=evidence, + ocr_tokens=tokens, + ) + output_is_raw = selection["output_state_sha256"] == selection["raw_state_sha256"] + output_is_candidate = selection["output_state_sha256"] == stable_sha256(candidate) + if true_edit: + final_counts["true_edit_candidate"] += int(output_is_candidate) + final_counts["true_edit_raw"] += int(output_is_raw) + else: + final_counts["true_keep_raw"] += int(output_is_raw) + final_counts["harmful_accepted"] += int(not output_is_raw) + reason_counts.update(selection["reason_codes"]) + if predicted_edit != (selection["decision"] == "ACCEPT"): + failures.append({ + "sample_id": record["sample_id"], + "predicted_edit": predicted_edit, + "selection_decision": selection["decision"], + "features": features, + "reason_codes": selection["reason_codes"], + }) + + tp, fp, tn, fn = (route_counts[key] for key in ("tp", "fp", "tn", "fn")) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + keep_preservation = tn / (tn + fp) if tn + fp else 0.0 + accepted_families = sum(values["tp"] > 0 for values in per_template.values()) + minimum_accepted_families = 2 if role == "development" else 4 + group_keep_trials = sum(values["tn"] + values["fp"] > 0 for values in group_routes.values()) + group_accept_trials = sum(values["tp"] + values["fn"] > 0 for values in group_routes.values()) + group_false_edits = sum(values["fp"] > 0 for values in group_routes.values()) + group_true_accepts = sum(values["tp"] > 0 for values in group_routes.values()) + harmful_upper = ( + _upper_zero_failures(group_keep_trials) + if group_false_edits == 0 else None + ) + precision_lower = ( + _lower_all_successes(group_true_accepts) + if group_false_edits == 0 else None + ) + gates = { + "raw_good_or_keep_preservation_ge_0_995": keep_preservation >= 0.995, + "harmful_takeover_group_upper_95_le_0_01": harmful_upper is not None and harmful_upper <= 0.01, + "takeover_precision_group_lower_95_ge_0_95": precision_lower is not None and precision_lower >= 0.95, + "nonzero_accept_coverage_meets_role_catalog": accepted_families >= minimum_accepted_families, + "selector_execution_consistent": not failures, + } + return { + "schema_version": "mpr-tsr/k1-geometry-selector-eval-v1", + "candidate_source": "controlled_counterfactual_bank_with_offline_gold_positive", + "candidate_generation_evaluated": False, + "role": role, + "records": selected_count, + "route_counts": dict(route_counts), + "route_precision": precision, + "route_recall": recall, + "keep_preservation": keep_preservation, + "harmful_takeover_group_upper_95": harmful_upper, + "all_success_accept_group_lower_95": precision_lower, + "independent_group_counts": { + "keep_trials": group_keep_trials, + "accept_trials": group_accept_trials, + "false_edit_groups": group_false_edits, + "true_accept_groups": group_true_accepts, + }, + "accepted_template_families": accepted_families, + "minimum_accepted_template_families": minimum_accepted_families, + "final_counts": dict(final_counts), + "selection_reason_counts": dict(reason_counts), + "gates": gates, + "status": "PASS" if all(gates.values()) else "FAIL", + "execution_failures": failures, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Evaluate the preregistered K1 token-geometry selector.") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--role", choices=("development", "holdout"), required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = evaluate(_load(args.input, args.role), args.role) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({ + "status": report["status"], + "role": report["role"], + "records": report["records"], + "route_counts": report["route_counts"], + "keep_preservation": report["keep_preservation"], + "all_success_accept_group_lower_95": report["all_success_accept_group_lower_95"], + "harmful_takeover_group_upper_95": report["harmful_takeover_group_upper_95"], + "accepted_template_families": report["accepted_template_families"], + }, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_raw_preserving_smoke.py b/scripts/run_raw_preserving_smoke.py new file mode 100644 index 0000000..735d723 --- /dev/null +++ b/scripts/run_raw_preserving_smoke.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import argparse +import json +from copy import deepcopy +from pathlib import Path + +from mpr_tsr_splitmerge_v2.counterfactual import build_candidate +from mpr_tsr_splitmerge_v2.raw_preserving import KEEP, label_oracle_action + + +def record(cells: list[tuple[int, int, str, list[int]]], *, cols: int) -> dict: + return { + "canonical_table": { + "rows": 1, + "cols": cols, + "cells": [ + { + "cell_id": f"c{index}", + "row": row, + "col": col, + "rowspan": 1, + "colspan": 1, + "text": text, + "tag": "td", + "bbox": [col * 10, 0, col * 10 + 9, 9], + "geometry": {"bbox": [col * 10, 0, col * 10 + 9, 9]}, + "ocr_token_indexes": token_indexes, + } + for index, (row, col, text, token_indexes) in enumerate(cells) + ], + }, + "non_table_state_sha256": "smoke-page-state", + "provenance": { + "sample_id": "raw-preserving-smoke", + "producer": "raw-preserving-smoke", + "producer_version": "v1", + "purpose": "nonterminal_logic_smoke", + "input_image_sha256": "0" * 64, + "terminal_benchmarks_visible": False, + }, + } + + +def build_cases(group_index: int) -> list[dict]: + token_text = [{"text": "A"}, {"text": "B"}] + raw_good = record([(0, 0, "A", [0]), (0, 1, "B", [1])], cols=2) + gold_good = deepcopy(raw_good) + raw_bad = record([(0, 0, "A", [0]), (0, 1, "WRONG", [1])], cols=2) + gold_bad = record([(0, 0, "A", [0]), (0, 1, "B", [1])], cols=2) + raw_merged = record([(0, 0, "AB", [0, 1])], cols=1) + gold_split = record([(0, 0, "A", [0]), (0, 1, "B", [1])], cols=2) + cases = [] + specifications = [ + (raw_good, gold_good, "identity", "raw_good_identity"), + (raw_good, gold_good, "over_merge", "raw_good_over_merge"), + (raw_good, gold_good, "over_merge", "raw_good_over_merge_2"), + (raw_bad, gold_bad, "identity", "raw_bad_identity"), + (raw_bad, gold_bad, "over_merge", "raw_bad_over_merge"), + (raw_bad, gold_bad, "over_merge", "raw_bad_over_merge_2"), + (raw_bad, gold_bad, "identity", "raw_bad_tie"), + (raw_merged, gold_split, "identity", "raw_bad_merged_identity"), + (raw_merged, gold_split, "over_split", "raw_bad_over_split"), + (raw_merged, gold_split, "over_merge", "raw_bad_over_merge_noop"), + ] + for local_index, (raw, gold, operator, tag) in enumerate(specifications): + candidate = build_candidate(raw, gold, operator=operator) + decision = label_oracle_action(raw, candidate, gold, ocr_tokens=token_text) + cases.append({ + "sample_id": f"smoke-{group_index:04d}-{local_index:02d}", + "pair_group_id": f"group-{group_index:04d}", + "image": {"path": f"synthetic://raw-preserving-smoke/{group_index:04d}.json"}, + "operator": operator, + "phenomenon_tags": [tag], + "oracle_decision": decision.as_dict(), + }) + return cases + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run the non-image Raw-preserving oracle smoke.") + parser.add_argument("--groups", type=int, default=256) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + if args.groups <= 0: + raise ValueError("groups must be positive") + cases = [case for group in range(args.groups) for case in build_cases(group)] + counts = {} + for case in cases: + action = case["oracle_decision"]["action"] + counts[action] = counts.get(action, 0) + 1 + expected_total = args.groups * 10 + if len(cases) != expected_total: + raise AssertionError(f"expected {expected_total} cases, got {len(cases)}") + if counts.get(KEEP, 0) != args.groups * 9: + raise AssertionError(f"expected 90% KEEP, got {counts}") + if counts.get("ACCEPT_EDIT", 0) != args.groups: + raise AssertionError(f"expected 10% ACCEPT_EDIT, got {counts}") + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + json.dumps({ + "schema_version": "mpr-tsr/raw-preserving-logic-smoke-v1", + "image_payload_included": False, + "groups": args.groups, + "records": len(cases), + "action_counts": counts, + "records_detail": cases, + }, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print(json.dumps({"groups": args.groups, "records": len(cases), "action_counts": counts}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/stress_token_geometry.py b/scripts/stress_token_geometry.py new file mode 100644 index 0000000..b19bb6f --- /dev/null +++ b/scripts/stress_token_geometry.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import random +from pathlib import Path +from typing import Any + +from mpr_tsr_splitmerge_v2.counterfactual import build_candidate +from mpr_tsr_splitmerge_v2.token_geometry import reassign_tokens_by_geometry + + +LEVELS = { + "none": 0.0, + "light": 0.02, + "medium": 0.08, + "heavy": 0.20, +} + + +def _load_groups(path: Path) -> list[dict[str, Any]]: + groups = {} + with path.open(encoding="utf-8") as handle: + for line in handle: + if line.strip(): + record = json.loads(line) + groups.setdefault(record["pair_group_id"], record) + return list(groups.values()) + + +def _jitter(tokens: list[dict[str, Any]], fraction: float, seed: int) -> list[dict[str, Any]]: + rng = random.Random(seed) + output = copy.deepcopy(tokens) + for token in output: + box = token.get("bbox") + if not isinstance(box, list) or len(box) != 4: + continue + width = float(box[2]) - float(box[0]) + height = float(box[3]) - float(box[1]) + dx = rng.uniform(-fraction, fraction) * width + dy = rng.uniform(-fraction, fraction) * height + token["bbox"] = [ + float(box[0]) + dx, + float(box[1]) + dy, + float(box[2]) + dx, + float(box[3]) + dy, + ] + return output + + +def evaluate(records: list[dict[str, Any]]) -> dict[str, Any]: + results = {} + for level, fraction in LEVELS.items(): + good_preserved = bad_fixed = rejected = 0 + for record in records: + gold = record["gold_record"] + seed = int(hashlib.sha256(f"{record['pair_group_id']}:{level}".encode()).hexdigest()[:16], 16) + tokens = _jitter(gold["ocr_tokens"], fraction, seed) + good_candidate = reassign_tokens_by_geometry(gold, tokens) + if good_candidate is None: + rejected += 1 + else: + good_preserved += good_candidate["canonical_table"] == gold["canonical_table"] + raw_bad = build_candidate(gold, gold, operator="assignment_swap") + bad_candidate = reassign_tokens_by_geometry(raw_bad, tokens) + if bad_candidate is None: + rejected += 1 + else: + bad_fixed += bad_candidate["canonical_table"] == gold["canonical_table"] + groups = len(records) + results[level] = { + "fraction": fraction, + "groups": groups, + "raw_good_preserved": good_preserved, + "raw_good_preservation": good_preserved / groups if groups else 0.0, + "raw_bad_fixed": bad_fixed, + "raw_bad_fix_rate": bad_fixed / groups if groups else 0.0, + "rejected_candidates": rejected, + } + light = results["light"] + gates = { + "light_raw_good_preservation_ge_0_995": light["raw_good_preservation"] >= 0.995, + "light_raw_bad_fix_rate_ge_0_95": light["raw_bad_fix_rate"] >= 0.95, + } + return { + "schema_version": "mpr-tsr/token-geometry-jitter-stress-v1", + "results": results, + "gates": gates, + "status": "PASS" if all(gates.values()) else "FAIL", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Stress token-geometry reassignment under preregistered bbox jitter.") + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + report = evaluate(_load_groups(args.input)) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"status": report["status"], "results": report["results"]}, ensure_ascii=False)) + return 0 if report["status"] == "PASS" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_synthetic_manifest.py b/scripts/validate_synthetic_manifest.py new file mode 100644 index 0000000..0fc117d --- /dev/null +++ b/scripts/validate_synthetic_manifest.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + + +FORBIDDEN_PATH_PARTS = {"..", "~"} + + +def iter_records(path: Path) -> list[dict[str, Any]]: + if path.suffix.lower() == ".jsonl": + records = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + else: + value = json.loads(path.read_text(encoding="utf-8")) + records = list(value) if isinstance(value, list) else list(value.values()) if isinstance(value, dict) else [] + if not records or not all(isinstance(record, dict) for record in records): + raise ValueError("manifest must contain one or more object records") + return records + + +def validate_manifest(manifest: Path, asset_root: Path) -> int: + records = iter_records(manifest) + checked = 0 + for index, record in enumerate(records): + image = record.get("image", record.get("file")) + if not isinstance(image, str) or not image: + raise ValueError(f"record {index} has no image/file reference") + image_path = Path(image) + if image_path.is_absolute() or any(part in FORBIDDEN_PATH_PARTS for part in image_path.parts): + raise ValueError(f"record {index} contains an unsafe image path: {image}") + if not (asset_root / image_path).is_file(): + raise ValueError(f"record {index} references a missing asset: {image}") + checked += 1 + return checked + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate a synthetic-data manifest without reading model outputs.") + parser.add_argument("manifest", type=Path) + parser.add_argument("asset_root", type=Path) + args = parser.parse_args() + checked = validate_manifest(args.manifest, args.asset_root) + print(f"validated_records={checked}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/mpr_tsr_splitmerge_v2/counterfactual.py b/src/mpr_tsr_splitmerge_v2/counterfactual.py new file mode 100644 index 0000000..6d89b8a --- /dev/null +++ b/src/mpr_tsr_splitmerge_v2/counterfactual.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import copy +from typing import Any + + +OPERATORS = ( + "identity", + "over_merge", + "over_split", + "assignment_swap", + "gold_candidate", +) + + +def _table(record: dict[str, Any]) -> dict[str, Any]: + table = record.get("canonical_table", record) + if not isinstance(table, dict) or not isinstance(table.get("cells"), list): + raise ValueError("record has no canonical table") + return table + + +def _new_record(raw: dict[str, Any], cells: list[dict[str, Any]], suffix: str) -> dict[str, Any]: + candidate = copy.deepcopy(raw) + table = _table(candidate) + table["cells"] = cells + provenance = candidate.setdefault("provenance", {}) + provenance["producer"] = "raw-preserving-counterfactual-bank" + provenance["producer_version"] = "v1" + provenance["purpose"] = suffix + return candidate + + +def _cell_bounds(cell: dict[str, Any]) -> tuple[int, int, int, int]: + if "row_start" in cell: + row = int(cell["row_start"]) + row_end = int(cell["row_end"]) + col = int(cell["col_start"]) + col_end = int(cell["col_end"]) + else: + row = int(cell.get("row", 0)) + row_end = row + int(cell.get("rowspan", 1) or 1) + col = int(cell.get("col", 0)) + col_end = col + int(cell.get("colspan", 1) or 1) + return row, row_end, col, col_end + + +def _set_bounds(cell: dict[str, Any], bounds: tuple[int, int, int, int]) -> None: + row, row_end, col, col_end = bounds + if "row_start" in cell: + cell.update(row_start=row, row_end=row_end, col_start=col, col_end=col_end) + else: + cell.update(row=row, col=col, rowspan=row_end - row, colspan=col_end - col) + + +def _merge_pair(cells: list[dict[str, Any]]) -> list[dict[str, Any]] | None: + for left_index, left in enumerate(cells): + lr, lre, lc, lce = _cell_bounds(left) + if lre - lr != 1 or lce - lc != 1: + continue + for right_index in range(left_index + 1, len(cells)): + right = cells[right_index] + rr, rre, rc, rce = _cell_bounds(right) + if (rr, rre, rc) != (lr, lre, lce): + continue + merged = copy.deepcopy(left) + _set_bounds(merged, (lr, lre, lc, rce)) + merged["text"] = f"{left.get('text', '')}{right.get('text', '')}" + if "ocr_token_indexes" in left or "ocr_token_indexes" in right: + merged["ocr_token_indexes"] = sorted( + list(left.get("ocr_token_indexes", [])) + + list(right.get("ocr_token_indexes", [])) + ) + left_box = left.get("bbox") + right_box = right.get("bbox") + if isinstance(left_box, list) and isinstance(right_box, list): + merged["bbox"] = [left_box[0], min(left_box[1], right_box[1]), right_box[2], max(left_box[3], right_box[3])] + if isinstance(merged.get("geometry"), dict): + merged["geometry"]["bbox"] = list(merged["bbox"]) + return [cell for index, cell in enumerate(cells) if index not in {left_index, right_index}] + [merged] + return None + + +def _split_cell(cells: list[dict[str, Any]]) -> list[dict[str, Any]] | None: + for index, cell in enumerate(cells): + row, row_end, col, col_end = _cell_bounds(cell) + tokens = list(cell.get("ocr_token_indexes", [])) + if row_end - row != 1 or col_end - col != 1 or len(tokens) < 2: + continue + midpoint = len(tokens) // 2 + first = copy.deepcopy(cell) + second = copy.deepcopy(cell) + first["cell_id"] = f"{cell.get('cell_id', index)}-left" + second["cell_id"] = f"{cell.get('cell_id', index)}-right" + _set_bounds(first, (row, row_end, col, col + 1)) + _set_bounds(second, (row, row_end, col + 1, col + 2)) + first["text"] = str(cell.get("text", ""))[: max(1, len(str(cell.get("text", ""))) // 2)] + second["text"] = str(cell.get("text", ""))[max(1, len(str(cell.get("text", ""))) // 2):] + first["ocr_token_indexes"] = tokens[:midpoint] + second["ocr_token_indexes"] = tokens[midpoint:] + return cells[:index] + [first, second] + cells[index + 1:] + return None + + +def _swap_assignment(cells: list[dict[str, Any]]) -> list[dict[str, Any]] | None: + candidates = [ + index + for index, cell in enumerate(cells) + if str(cell.get("text", "")) and cell.get("ocr_token_indexes") + ] + if len(candidates) < 2: + return None + left_index, right_index = candidates[0], candidates[1] + output = copy.deepcopy(cells) + left = output[left_index] + right = output[right_index] + left["text"], right["text"] = right["text"], left["text"] + left["ocr_token_indexes"], right["ocr_token_indexes"] = ( + list(right["ocr_token_indexes"]), + list(left["ocr_token_indexes"]), + ) + return output + + +def build_candidate( + raw_record: dict[str, Any], + gold_record: dict[str, Any], + *, + operator: str, +) -> dict[str, Any] | None: + """Build a deterministic nonterminal candidate for smoke and QA only.""" + if operator not in OPERATORS: + raise ValueError(f"unknown operator: {operator}") + if operator == "identity": + return copy.deepcopy(raw_record) + if operator == "gold_candidate": + candidate = copy.deepcopy(gold_record) + candidate.setdefault("provenance", {})["purpose"] = "nonterminal_oracle_candidate_smoke_only" + return candidate + cells = copy.deepcopy(_table(raw_record)["cells"]) + if operator == "over_merge": + changed = _merge_pair(cells) + elif operator == "over_split": + changed = _split_cell(cells) + else: + changed = _swap_assignment(cells) + if changed is None: + return None + candidate = _new_record(raw_record, changed, f"nonterminal_{operator}_smoke_only") + if operator == "over_split": + candidate["canonical_table"]["cols"] = int(candidate["canonical_table"].get("cols", 0)) + 1 + return candidate diff --git a/src/mpr_tsr_splitmerge_v2/otsl.py b/src/mpr_tsr_splitmerge_v2/otsl.py new file mode 100644 index 0000000..d5d5b23 --- /dev/null +++ b/src/mpr_tsr_splitmerge_v2/otsl.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import re + + +CONTROL_TOKENS = ("<|im_end|>", "<|endoftext|>", "<|im_start|>") +STRUCTURE_TOKENS = ("", "", "", "", "", "") + + +def to_native_otsl(text: str) -> str: + """Normalize OTSL row boundaries to the literal-newline dialect.""" + normalized = text.replace("\n", "") + return normalized.replace("", "\n").rstrip("\n") + + +def strip_chat_controls(text: str) -> str: + """Remove only chat wrapper tokens after structure-preserving decoding.""" + for token in CONTROL_TOKENS: + text = text.replace(token, "") + return text + + +def validate_structure_tokens(text: str) -> None: + """Reject missing or non-atomic structural markers in a decoded OTSL string.""" + if not any(token in text for token in STRUCTURE_TOKENS): + raise ValueError("decoded output contains no OTSL structure token") + if "\n" not in text and "" in text: + raise ValueError("OTSL newline token is not followed by a literal newline") + + +def normalize_decoded_otsl(text: str) -> str: + """Apply the safe post-decode normalization used by the public examples.""" + normalized = to_native_otsl(strip_chat_controls(text)) + validate_structure_tokens(normalized) + return normalized + + +def count_structure_tokens(text: str) -> dict[str, int]: + """Count exact OTSL markers without interpreting cell text.""" + return {token: len(re.findall(re.escape(token), text)) for token in STRUCTURE_TOKENS} diff --git a/src/mpr_tsr_splitmerge_v2/paired_metrics.py b/src/mpr_tsr_splitmerge_v2/paired_metrics.py new file mode 100644 index 0000000..4a405fe --- /dev/null +++ b/src/mpr_tsr_splitmerge_v2/paired_metrics.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass + + +@dataclass(frozen=True) +class PairedSummary: + baseline_correct: int + candidate_correct: int + fixed_denominator: int + delta_correct: int + delta_pp: float + help_count: int + hurt_count: int + same_count: int + largest_absolute_delta: int + leave_one_out_delta_pp: float + + +def summarize_paired( + baseline: Sequence[tuple[int, int]], + candidate: Sequence[tuple[int, int]], +) -> PairedSummary: + """Summarize paired fixed-denominator cell results. + + Each item is ``(correct_cells, fixed_gt_denominator)``. The two sequences + must describe the same examples in the same order. + """ + if not baseline or len(baseline) != len(candidate): + raise ValueError("paired results must be non-empty and equally sized") + pairs = list(zip(baseline, candidate)) + if any(base_den <= 0 or candidate_den <= 0 for (_, base_den), (_, candidate_den) in pairs): + raise ValueError("fixed denominators must be positive") + if any(base_den != candidate_den for (_, base_den), (_, candidate_den) in pairs): + raise ValueError("baseline and candidate denominators must match") + + deltas = [candidate_item[0] - base_item[0] for base_item, candidate_item in pairs] + denominator = sum(den for _, den in baseline) + baseline_correct = sum(correct for correct, _ in baseline) + candidate_correct = sum(correct for correct, _ in candidate) + largest_index = max(range(len(deltas)), key=lambda index: abs(deltas[index])) + largest = deltas[largest_index] + loo_denominator = denominator - baseline[largest_index][1] + if loo_denominator <= 0: + raise ValueError("leave-one-out denominator must be positive") + loo_delta = candidate_correct - baseline_correct - largest + return PairedSummary( + baseline_correct=baseline_correct, + candidate_correct=candidate_correct, + fixed_denominator=denominator, + delta_correct=candidate_correct - baseline_correct, + delta_pp=100.0 * (candidate_correct - baseline_correct) / denominator, + help_count=sum(delta > 0 for delta in deltas), + hurt_count=sum(delta < 0 for delta in deltas), + same_count=sum(delta == 0 for delta in deltas), + largest_absolute_delta=largest, + leave_one_out_delta_pp=100.0 * loo_delta / loo_denominator, + ) diff --git a/src/mpr_tsr_splitmerge_v2/raw_preserving.py b/src/mpr_tsr_splitmerge_v2/raw_preserving.py new file mode 100644 index 0000000..25a4076 --- /dev/null +++ b/src/mpr_tsr_splitmerge_v2/raw_preserving.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +import copy +from collections import Counter +from dataclasses import asdict, dataclass +from typing import Any + +from .canonical import normalize_text +from .safety_layer import SafetyPolicy, validate_candidate + + +KEEP = "KEEP_RAW" +EDIT = "ACCEPT_EDIT" + +REASONS = { + "RAW_ALREADY_CORRECT", + "CANDIDATE_MISSING", + "CANDIDATE_INVALID", + "NO_STRICT_GAIN", + "TEXT_REGRESSION", + "GEOMETRY_REGRESSION", + "NON_TABLE_STATE_CHANGED", + "STRICTLY_BETTER_CANDIDATE", +} + + +@dataclass(frozen=True) +class OracleDecision: + action: str + reason: str + raw_good: bool + candidate_valid: bool + gold_cell_denominator: int + raw_exact_cells: int + candidate_exact_cells: int + raw_text_exact: int + candidate_text_exact: int + raw_geometry_coverage: int + candidate_geometry_coverage: int + delta_exact_cells: int + delta_pp: float + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +def _cells(record: dict[str, Any]) -> list[dict[str, Any]]: + table = record.get("canonical_table", record) + if not isinstance(table, dict) or not isinstance(table.get("cells"), list): + return [] + output = [] + for value in table["cells"]: + if not isinstance(value, dict): + continue + if "row_start" in value: + row = int(value.get("row_start", 0)) + col = int(value.get("col_start", 0)) + rowspan = int(value.get("row_end", row + 1)) - row + colspan = int(value.get("col_end", col + 1)) - col + else: + row = int(value.get("row", 0)) + col = int(value.get("col", 0)) + rowspan = int(value.get("rowspan", 1) or 1) + colspan = int(value.get("colspan", 1) or 1) + output.append({ + "row": row, + "col": col, + "rowspan": rowspan, + "colspan": colspan, + "text": str(value.get("text", "") or ""), + "tag": "th" if str(value.get("tag", "td")).lower() == "th" else "td", + "bbox": value.get("bbox", value.get("geometry", {}).get("bbox") if isinstance(value.get("geometry"), dict) else None), + }) + return output + + +def _strict_key(cell: dict[str, Any]) -> tuple[Any, ...]: + return ( + cell["row"], cell["col"], cell["rowspan"], cell["colspan"], + normalize_text(cell["text"]), cell["tag"], + ) + + +def _cell_exact_count( + predicted: list[dict[str, Any]], gold: list[dict[str, Any]] +) -> int: + remaining = Counter(_strict_key(cell) for cell in gold) + correct = 0 + for cell in predicted: + key = _strict_key(cell) + if remaining[key] > 0: + remaining[key] -= 1 + correct += 1 + return correct + + +def _text_exact_count( + predicted: list[dict[str, Any]], gold: list[dict[str, Any]] +) -> int: + remaining = Counter(normalize_text(cell.get("text", "")) for cell in gold) + correct = 0 + for cell in predicted: + text = normalize_text(cell.get("text", "")) + if remaining[text] > 0: + remaining[text] -= 1 + correct += 1 + return correct + + +def _geometry_coverage(cells: list[dict[str, Any]]) -> int: + count = 0 + for cell in cells: + geometry = cell.get("geometry") + bbox = geometry.get("bbox") if isinstance(geometry, dict) else cell.get("bbox") + if isinstance(bbox, list) and len(bbox) == 4: + try: + if float(bbox[2]) > float(bbox[0]) and float(bbox[3]) > float(bbox[1]): + count += 1 + except (TypeError, ValueError): + pass + return count + + +def _non_table_state_changed( + raw_record: dict[str, Any], candidate_record: dict[str, Any] +) -> bool: + raw_state = raw_record.get("non_table_state_sha256") + if raw_state is None: + return False + return candidate_record.get("non_table_state_sha256") != raw_state + + +def _validate( + raw_record: dict[str, Any], + candidate_record: dict[str, Any] | None, + *, + ocr_tokens: list[dict[str, Any]] | None, + policy: SafetyPolicy, +) -> bool: + if candidate_record is None: + return False + return validate_candidate( + raw_record, + candidate_record, + policy=policy, + ocr_tokens=ocr_tokens, + )["status"] == "PASS" + + +def _decision( + *, + action: str, + reason: str, + raw_good: bool, + candidate_valid: bool, + denominator: int, + raw_cells: list[dict[str, Any]], + candidate_cells: list[dict[str, Any]], + gold_cells: list[dict[str, Any]], +) -> OracleDecision: + raw_exact = _cell_exact_count(raw_cells, gold_cells) + candidate_exact = _cell_exact_count(candidate_cells, gold_cells) + raw_text = _text_exact_count(raw_cells, gold_cells) + candidate_text = _text_exact_count(candidate_cells, gold_cells) + delta = candidate_exact - raw_exact + return OracleDecision( + action=action, + reason=reason, + raw_good=raw_good, + candidate_valid=candidate_valid, + gold_cell_denominator=denominator, + raw_exact_cells=raw_exact, + candidate_exact_cells=candidate_exact, + raw_text_exact=raw_text, + candidate_text_exact=candidate_text, + raw_geometry_coverage=_geometry_coverage(raw_cells), + candidate_geometry_coverage=_geometry_coverage(candidate_cells), + delta_exact_cells=delta, + delta_pp=100.0 * delta / denominator if denominator else 0.0, + ) + + +def label_oracle_action( + raw_record: dict[str, Any], + candidate_record: dict[str, Any] | None, + gold_record: dict[str, Any], + *, + ocr_tokens: list[dict[str, Any]] | None = None, + policy: SafetyPolicy | None = None, +) -> OracleDecision: + """Create an offline KEEP/EDIT label from Raw, candidate, and Gold. + + Gold is required only for this offline function. The runtime selector view + deliberately excludes Gold and every Gold-derived metric. + """ + raw_cells = _cells(raw_record) + gold_cells = _cells(gold_record) + candidate_cells = _cells(candidate_record or {}) + denominator = len(gold_cells) + raw_exact = _cell_exact_count(raw_cells, gold_cells) + raw_good = bool(gold_cells) and len(raw_cells) == denominator and raw_exact == denominator + validation_policy = policy or SafetyPolicy( + policy_id="offline-oracle-v1", + minimum_expected_gain=0.0, + threshold_source="NONTERMINAL_PREREGISTERED", + text_policy="OCR_GROUNDED" if ocr_tokens is not None else "FROZEN_RAW", + ) + candidate_valid = _validate( + raw_record, + candidate_record, + ocr_tokens=ocr_tokens, + policy=validation_policy, + ) + if raw_good: + return _decision( + action=KEEP, + reason="RAW_ALREADY_CORRECT", + raw_good=True, + candidate_valid=candidate_valid, + denominator=denominator, + raw_cells=raw_cells, + candidate_cells=candidate_cells, + gold_cells=gold_cells, + ) + + if candidate_record is None: + return _decision( + action=KEEP, + reason="CANDIDATE_MISSING", + raw_good=False, + candidate_valid=False, + denominator=denominator, + raw_cells=raw_cells, + candidate_cells=[], + gold_cells=gold_cells, + ) + + validation = validate_candidate( + raw_record, + candidate_record, + policy=validation_policy, + ocr_tokens=ocr_tokens, + ) + if validation["status"] != "PASS": + return _decision( + action=KEEP, + reason="CANDIDATE_INVALID", + raw_good=False, + candidate_valid=False, + denominator=denominator, + raw_cells=raw_cells, + candidate_cells=candidate_cells, + gold_cells=gold_cells, + ) + + raw_text = _text_exact_count(raw_cells, gold_cells) + candidate_text = _text_exact_count(candidate_cells, gold_cells) + raw_geometry = _geometry_coverage(raw_cells) + candidate_geometry = _geometry_coverage(candidate_cells) + raw_exact = _cell_exact_count(raw_cells, gold_cells) + candidate_exact = _cell_exact_count(candidate_cells, gold_cells) + if _non_table_state_changed(raw_record, candidate_record): + reason = "NON_TABLE_STATE_CHANGED" + elif candidate_text < raw_text: + reason = "TEXT_REGRESSION" + elif candidate_geometry < raw_geometry: + reason = "GEOMETRY_REGRESSION" + elif candidate_exact <= raw_exact: + reason = "NO_STRICT_GAIN" + else: + reason = "STRICTLY_BETTER_CANDIDATE" + return _decision( + action=EDIT if reason == "STRICTLY_BETTER_CANDIDATE" else KEEP, + reason=reason, + raw_good=False, + candidate_valid=True, + denominator=denominator, + raw_cells=raw_cells, + candidate_cells=candidate_cells, + gold_cells=gold_cells, + ) + + +def _runtime_cell(value: dict[str, Any]) -> dict[str, Any]: + allowed = ( + "cell_id", "row", "col", "rowspan", "colspan", "row_start", + "row_end", "col_start", "col_end", "text", "tag", + "ocr_token_indexes", "geometry", "bbox", + ) + return {key: copy.deepcopy(value[key]) for key in allowed if key in value} + + +def _runtime_record(record: dict[str, Any]) -> dict[str, Any]: + table = record.get("canonical_table", record) + if not isinstance(table, dict): + return {"canonical_table": {"cells": []}} + output = {"canonical_table": { + key: copy.deepcopy(table[key]) + for key in ("rows", "cols") + if key in table + }} + output["canonical_table"]["cells"] = [ + _runtime_cell(value) + for value in table.get("cells", []) + if isinstance(value, dict) + ] + return output + + +def _runtime_image(image: dict[str, Any]) -> dict[str, Any]: + return { + key: copy.deepcopy(image[key]) + for key in ("path", "width", "height", "sha256", "image_sha256") + if key in image + } + + +def selector_runtime_view( + *, + sample_id: str, + image: dict[str, Any], + raw_record: dict[str, Any], + candidate_record: dict[str, Any], +) -> dict[str, Any]: + """Build the inference-time whitelist; Gold-derived fields cannot enter.""" + raw_cells = _cells(raw_record) + candidate_cells = _cells(candidate_record) + return { + "schema_version": "mpr-tsr/raw-preserving-selector-view-v1", + "sample_id": sample_id, + "image": _runtime_image(image), + "raw_record": _runtime_record(raw_record), + "candidate_record": _runtime_record(candidate_record), + "candidate_diff": { + "raw_cell_count": len(raw_cells), + "candidate_cell_count": len(candidate_cells), + "cell_count_delta": len(candidate_cells) - len(raw_cells), + "raw_rows": raw_record.get("canonical_table", {}).get("rows"), + "candidate_rows": candidate_record.get("canonical_table", {}).get("rows"), + "raw_cols": raw_record.get("canonical_table", {}).get("cols"), + "candidate_cols": candidate_record.get("canonical_table", {}).get("cols"), + }, + } diff --git a/src/mpr_tsr_splitmerge_v2/token_geometry.py b/src/mpr_tsr_splitmerge_v2/token_geometry.py new file mode 100644 index 0000000..1e4ee64 --- /dev/null +++ b/src/mpr_tsr_splitmerge_v2/token_geometry.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import copy +from typing import Any + + +def _bbox(value: dict[str, Any]) -> list[float] | None: + geometry = value.get("geometry") + box = geometry.get("bbox") if isinstance(geometry, dict) else value.get("bbox") + if not isinstance(box, list) or len(box) != 4: + return None + return [float(item) for item in box] + + +def _contains(box: list[float], center: tuple[float, float]) -> bool: + return box[0] <= center[0] <= box[2] and box[1] <= center[1] <= box[3] + + +def topology_signature(record: dict[str, Any]) -> tuple[Any, ...]: + table = record.get("canonical_table", record) + cells = table.get("cells", []) if isinstance(table, dict) else [] + return ( + table.get("rows"), + table.get("cols"), + tuple(sorted( + ( + cell.get("row", cell.get("row_start")), + cell.get("col", cell.get("col_start")), + cell.get("rowspan", cell.get("row_end")), + cell.get("colspan", cell.get("col_end")), + cell.get("tag", "td"), + ) + for cell in cells + )), + ) + + +def ownership_violations( + record: dict[str, Any], + ocr_tokens: list[dict[str, Any]], +) -> int | None: + table = record.get("canonical_table", record) + cells = table.get("cells", []) if isinstance(table, dict) else [] + violations = 0 + seen = set() + for cell in cells: + cell_box = _bbox(cell) + if cell_box is None: + return None + indexes = cell.get("ocr_token_indexes") + if not isinstance(indexes, list): + return None + for index in indexes: + if not isinstance(index, int) or index < 0 or index >= len(ocr_tokens) or index in seen: + return None + token_box = _bbox(ocr_tokens[index]) + if token_box is None: + return None + seen.add(index) + center = ((token_box[0] + token_box[2]) / 2, (token_box[1] + token_box[3]) / 2) + if not _contains(cell_box, center): + violations += 1 + return violations + + +def reassign_tokens_by_geometry( + raw_record: dict[str, Any], + ocr_tokens: list[dict[str, Any]], +) -> dict[str, Any] | None: + candidate = copy.deepcopy(raw_record) + table = candidate.get("canonical_table", candidate) + cells = table.get("cells", []) if isinstance(table, dict) else [] + cell_boxes = [_bbox(cell) for cell in cells] + if any(box is None for box in cell_boxes): + return None + assignments: list[list[int]] = [[] for _ in cells] + for index, token in enumerate(ocr_tokens): + token_box = _bbox(token) + if token_box is None: + return None + center = ((token_box[0] + token_box[2]) / 2, (token_box[1] + token_box[3]) / 2) + owners = [cell_index for cell_index, box in enumerate(cell_boxes) if _contains(box, center)] + if len(owners) != 1: + return None + assignments[owners[0]].append(index) + for cell, indexes in zip(cells, assignments): + indexes.sort() + cell["ocr_token_indexes"] = indexes + cell["text"] = "".join(str(ocr_tokens[index].get("text", "")) for index in indexes) + provenance = candidate.setdefault("provenance", {}) + provenance["producer"] = "token-geometry-reassignment" + provenance["producer_version"] = "v1" + provenance["purpose"] = "observable_token_assignment_candidate" + return candidate diff --git a/tests/test_public_research_tools.py b/tests/test_public_research_tools.py new file mode 100644 index 0000000..b5b9f74 --- /dev/null +++ b/tests/test_public_research_tools.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from mpr_tsr_splitmerge_v2.otsl import ( + count_structure_tokens, + normalize_decoded_otsl, + to_native_otsl, +) +from mpr_tsr_splitmerge_v2.paired_metrics import summarize_paired + + +def test_native_otsl_preserves_one_literal_newline_per_row_boundary() -> None: + assert to_native_otsl("A\nB") == "A\nB" + + +def test_decoded_otsl_removes_only_chat_controls() -> None: + text = normalize_decoded_otsl("<|im_end|>AB") + assert text == "A\nB" + assert count_structure_tokens(text)[""] == 2 + + +def test_missing_structure_tokens_are_rejected() -> None: + try: + normalize_decoded_otsl("plain text") + except ValueError as error: + assert "structure token" in str(error) + else: + raise AssertionError("plain text was accepted as OTSL") + + +def test_paired_metrics_uses_the_removed_item_denominator_for_loo() -> None: + baseline = [(8, 10), (90, 100)] + candidate = [(10, 10), (89, 100)] + summary = summarize_paired(baseline, candidate) + assert summary.delta_correct == 1 + assert summary.largest_absolute_delta == 2 + assert summary.leave_one_out_delta_pp == -1.0 diff --git a/tests/test_raw_preserving_oracle.py b/tests/test_raw_preserving_oracle.py new file mode 100644 index 0000000..27ab8b5 --- /dev/null +++ b/tests/test_raw_preserving_oracle.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from copy import deepcopy + +from mpr_tsr_splitmerge_v2.raw_preserving import ( + EDIT, + KEEP, + label_oracle_action, + selector_runtime_view, +) + + +def _record(values): + cells = [] + for index, (row, col, text) in enumerate(values): + cells.append({ + "cell_id": f"c{index}", + "row": row, + "col": col, + "rowspan": 1, + "colspan": 1, + "text": text, + "tag": "td", + "bbox": [col * 10, row * 10, col * 10 + 9, row * 10 + 9], + "geometry": {"bbox": [col * 10, row * 10, col * 10 + 9, row * 10 + 9]}, + "ocr_token_indexes": [index], + }) + return { + "canonical_table": {"rows": 1, "cols": len(cells), "cells": cells}, + "non_table_state_sha256": "same-page-state", + "provenance": { + "sample_id": "fixture", + "producer": "fixture", + "producer_version": "v1", + "purpose": "nonterminal_fixture", + "input_image_sha256": "0" * 64, + "terminal_benchmarks_visible": False, + }, + } + + +def test_raw_good_forces_keep_even_with_a_valid_candidate(): + gold = _record([(0, 0, "A"), (0, 1, "B")]) + candidate = deepcopy(gold) + decision = label_oracle_action(gold, candidate, gold) + assert decision.action == KEEP + assert decision.reason == "RAW_ALREADY_CORRECT" + assert decision.raw_good is True + + +def test_strictly_better_candidate_is_accept_edit(): + raw = _record([(0, 0, "A"), (0, 1, "WRONG")]) + gold = _record([(0, 0, "A"), (0, 1, "B")]) + candidate = deepcopy(gold) + decision = label_oracle_action(raw, candidate, gold, ocr_tokens=[{"text": "A"}, {"text": "B"}]) + assert decision.action == EDIT + assert decision.reason == "STRICTLY_BETTER_CANDIDATE" + assert decision.delta_exact_cells == 1 + + +def test_legal_but_tied_candidate_stays_keep(): + raw = _record([(0, 0, "A"), (0, 1, "WRONG")]) + gold = _record([(0, 0, "A"), (0, 1, "B")]) + candidate = deepcopy(raw) + decision = label_oracle_action(raw, candidate, gold) + assert decision.action == KEEP + assert decision.reason == "NO_STRICT_GAIN" + + +def test_runtime_view_excludes_gold_and_gold_derived_fields(): + raw = _record([(0, 0, "A")]) + candidate = deepcopy(raw) + view = selector_runtime_view( + sample_id="s1", + image={"path": "sample.jpg"}, + raw_record=raw, + candidate_record=candidate, + ) + serialized = str(view) + assert "gold" not in serialized.lower() + assert "raw_score" not in serialized + assert "candidate_score" not in serialized + assert "decision_target" not in serialized diff --git a/tests/test_token_geometry.py b/tests/test_token_geometry.py new file mode 100644 index 0000000..56835e5 --- /dev/null +++ b/tests/test_token_geometry.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from copy import deepcopy + +from mpr_tsr_splitmerge_v2.counterfactual import build_candidate +from mpr_tsr_splitmerge_v2.token_geometry import ( + ownership_violations, + reassign_tokens_by_geometry, + topology_signature, +) + + +def _record() -> dict: + tokens = [ + {"text": "A", "bbox": [0.0, 0.0, 10.0, 10.0]}, + {"text": "B", "bbox": [10.0, 0.0, 20.0, 10.0]}, + ] + return { + "canonical_table": { + "rows": 1, + "cols": 2, + "cells": [ + { + "cell_id": "c0", + "row": 0, + "col": 0, + "rowspan": 1, + "colspan": 1, + "text": "A", + "tag": "td", + "bbox": [0.0, 0.0, 10.0, 10.0], + "geometry": {"bbox": [0.0, 0.0, 10.0, 10.0]}, + "ocr_token_indexes": [0], + }, + { + "cell_id": "c1", + "row": 0, + "col": 1, + "rowspan": 1, + "colspan": 1, + "text": "B", + "tag": "td", + "bbox": [10.0, 0.0, 20.0, 10.0], + "geometry": {"bbox": [10.0, 0.0, 20.0, 10.0]}, + "ocr_token_indexes": [1], + }, + ], + }, + "ocr_tokens": tokens, + "non_table_state_sha256": "same", + "provenance": {}, + } + + +def test_assignment_swap_preserves_coarse_structure_and_text_multiset() -> None: + gold = _record() + swapped = build_candidate(gold, gold, operator="assignment_swap") + assert swapped is not None + assert topology_signature(swapped) == topology_signature(gold) + assert len(swapped["canonical_table"]["cells"]) == len(gold["canonical_table"]["cells"]) + assert sorted(cell["text"] for cell in swapped["canonical_table"]["cells"]) == ["A", "B"] + assert swapped["canonical_table"] != gold["canonical_table"] + + +def test_geometry_reassignment_preserves_good_and_repairs_swapped_assignment() -> None: + gold = _record() + tokens = gold["ocr_tokens"] + identity = reassign_tokens_by_geometry(gold, tokens) + assert identity is not None + assert identity["canonical_table"] == gold["canonical_table"] + swapped = build_candidate(gold, gold, operator="assignment_swap") + assert swapped is not None + assert ownership_violations(swapped, tokens) == 2 + repaired = reassign_tokens_by_geometry(swapped, tokens) + assert repaired is not None + assert ownership_violations(repaired, tokens) == 0 + assert repaired["canonical_table"] == gold["canonical_table"]