From 5c5112d212b6d4ba76c9088fe15cec850dc77926 Mon Sep 17 00:00:00 2001 From: Karl Date: Wed, 12 Aug 2026 12:00:44 +0800 Subject: [PATCH] Establish research snapshot 2026.08.12 --- .gitignore | 4 + CONTRIBUTING.md | 114 +++- README.md | 308 ++++++----- dataset/README.md | 5 +- docs/COLLABORATOR_HANDOFF.md | 8 +- docs/DATASET_STORAGE_AND_SHARING.md | 8 +- docs/DATA_GOVERNANCE.md | 10 +- docs/REPRODUCIBILITY.md | 18 +- docs/SYNTHETIC_CORPUS_SPECIFICATION.md | 8 +- docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md | 93 ++++ ...ENCE_CARD_CANONICAL_DATA_LAYER_VNEXT_V1.md | 87 --- .../EVIDENCE_CARD_SHARED_SAFETY_LAYER_V1.md | 65 --- ...NCE_CARD_STAGE3_CANDIDATE_INTERFACES_V1.md | 121 ----- ...IDENCE_CARD_STAGE4_CORRECTNESS_SMOKE_V2.md | 50 -- .../POST_OMNIDOCBENCH_CONTRACT_V6_SEAL.json | 30 - ...POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md | 513 ------------------ pyproject.toml | 4 +- ...=> canonical_table_record_2026.08.12.json} | 8 +- scripts/compile_canonical_data_layer_vnext.py | 427 --------------- src/borderless_table_structuring/__init__.py | 5 + .../candidate_integrity.py | 11 +- .../candidate_interfaces.py | 14 +- .../canonical.py | 0 src/borderless_table_structuring/explicit.py | 20 + .../labels.py | 10 +- .../safety_layer.py | 16 +- src/mpr_tsr_splitmerge_v2/__init__.py | 3 - tests/test_candidate_integrity.py | 16 +- tests/test_candidate_interfaces.py | 12 +- tests/test_canonical_data_layer_vnext.py | 65 --- tests/test_explicit_route.py | 77 +++ tests/test_safety_layer.py | 20 +- 32 files changed, 523 insertions(+), 1627 deletions(-) create mode 100644 docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md delete mode 100644 evidence/EVIDENCE_CARD_CANONICAL_DATA_LAYER_VNEXT_V1.md delete mode 100644 evidence/EVIDENCE_CARD_SHARED_SAFETY_LAYER_V1.md delete mode 100644 evidence/EVIDENCE_CARD_STAGE3_CANDIDATE_INTERFACES_V1.md delete mode 100644 evidence/EVIDENCE_CARD_STAGE4_CORRECTNESS_SMOKE_V2.md delete mode 100644 governance/POST_OMNIDOCBENCH_CONTRACT_V6_SEAL.json delete mode 100644 governance/POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md rename schemas/{canonical_data_layer_vnext_schema_v1.json => canonical_table_record_2026.08.12.json} (85%) delete mode 100644 scripts/compile_canonical_data_layer_vnext.py create mode 100644 src/borderless_table_structuring/__init__.py rename src/{mpr_tsr_splitmerge_v2 => borderless_table_structuring}/candidate_integrity.py (97%) rename src/{mpr_tsr_splitmerge_v2 => borderless_table_structuring}/candidate_interfaces.py (95%) rename src/{mpr_tsr_splitmerge_v2 => borderless_table_structuring}/canonical.py (100%) create mode 100644 src/borderless_table_structuring/explicit.py rename src/{mpr_tsr_splitmerge_v2 => borderless_table_structuring}/labels.py (98%) rename src/{mpr_tsr_splitmerge_v2 => borderless_table_structuring}/safety_layer.py (96%) delete mode 100644 src/mpr_tsr_splitmerge_v2/__init__.py delete mode 100644 tests/test_canonical_data_layer_vnext.py create mode 100644 tests/test_explicit_route.py diff --git a/.gitignore b/.gitignore index f46d050..2b3e3aa 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ runs/ *.key credentials.json token.json + +# Internal coordination records stay in the parent workspace, not this lab repo. +governance/ +evidence/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 64283af..acc55f9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,40 +1,100 @@ # Contributing -## Before starting +Thank you for contributing to Borderless Table Structuring Lab. The repository +supports collaborative research on table representations, data generation, +explicit layout reasoning, and parameter-efficient generative modeling. -Read the active execution contract and its seal in `governance/` in full. A -contribution must stay inside the current authorization boundary. +## Shared research method -## Branch and review workflow +All model tracks operate on the same Canonical Table representation of logical +topology, physical geometry, cell content, and OCR-token ownership. -1. Create a focused branch from the current default branch. -2. Declare the hypothesis and single variable in the pull request. -3. Add or update deterministic tests. -4. Record source, license, split role, and isolation implications. -5. Add an Evidence Card for any data-affecting stage. -6. Request review before merging. +- **Explicit Layout Transformer** predicts sparse, order-invariant topology + changes. Structural proposals remain separable from text and geometry. +- **LoRA Table Model** predicts one complete Canonical Table state through + parameter-efficient adaptation. +- **Shared data and evaluation** provide identical document roles, schemas, + OCR evidence, geometry, metrics, and overlap audits for both tracks. + +Keep the two model implementations independent until a comparison explicitly +studies their interaction. Shared infrastructure belongs in neutral modules; +track-specific assumptions belong under the corresponding method or model +area. + +## Calendar versioning + +Project-authored snapshots, models, datasets, schemas, experiment records, and +evaluation artifacts use calendar versioning without a `v` prefix. + +| Artifact | Preferred identifier | Same-day revision | +|---|---|---| +| Research snapshot or Git tag | `2026.08.12` | `2026.08.12.1` | +| Explicit model | `explicit-2026.08.12` | `explicit-2026.08.12.1` | +| LoRA model | `lora-2026.08.12` | `lora-2026.08.12.1` | +| Dataset release | `data-2026.08.12` | `data-2026.08.12.1` | +| Schema release | `canonical-table-record-2026.08.12` | `canonical-table-record-2026.08.12.1` | +| Experiment | `experiment-2026.08.12-` | append `.1` when needed | + +Use the most precise meaningful date. The identifier does not claim that every +artifact was created at exactly the same time; its purpose is to distinguish +research states reproducibly. Upstream packages, datasets, and benchmarks keep +their official external version names. + +## Branches and pull requests + +Use a focused branch from the current default branch: + +- `explicit/` for explicit topology modeling; +- `lora/` for generative adaptation and ablations; +- `data/` for corpus construction and validation; +- `eval/` for route-independent evaluation; +- `docs/` for research documentation. + +A pull request should state: + +1. the research question or engineering problem; +2. the hypothesis and changed variable; +3. the affected CalVer release; +4. the data roles, sources, licenses, and isolation implications; +5. the metrics and acceptance criteria; +6. the commands used for validation. + +Use an imperative English commit summary and keep each commit scoped to one +logical change, for example `Add deterministic span-template generator`. + +## Data contributions + +Data-related contributions must include source and license provenance, +deterministic seeds, family-level split assignments, record and payload hashes, +complete generated/pass/quarantine/failure counts, and exact plus near-duplicate +audits. Dataset payloads remain outside Git; commit schemas, generators, +manifests, and small synthetic fixtures only. + +Do not use one ordered split/merge action path as the sole target when multiple +paths produce the same table. Prefer the final Canonical Table state and an +order-invariant structural difference. + +## Model contributions + +Model code must declare its input representation, output contract, model CalVer +identifier, checkpoint provenance, and supported metrics. Add a small +deterministic correctness test before any performance experiment. Explicit and +LoRA results should be reported against the same frozen data roles and +route-independent metrics. ## Required checks - `pytest` passes. -- No dataset payload, model weight, credential, or terminal artifact is added. -- All new filenames, documentation, code comments, and manifests are English. +- All project-authored files, filenames, documentation, and comments are English. +- No dataset payload, model weight, credential, or private evaluation artifact + is committed. - Generated outputs use new, non-overwriting paths. - SHA256 manifests and complete failure accounting are present where required. -- Exact and near-duplicate audits pass before a corpus role is frozen. - -## Prohibited content - -Do not commit Customer50 or OmniDocBench terminal pages, crops, annotations, -recognized strings, coordinates, HTML, LaTeX, identifiers, Gold, predictions, -or derived near-duplicates. Do not commit source datasets unless their -redistribution and downstream-use rights have been explicitly approved. - -## Commit messages +- Exact and near-duplicate audits pass before a corpus role is released. -Use an imperative English summary and keep each commit scoped to one logical -change. Example: +## Restricted content -```text -Add deterministic span-template generator -``` +Do not commit restricted benchmark pages, private evaluation material, +annotations without redistribution rights, model weights, credentials, or +derived near-duplicates. Do not commit source datasets unless redistribution +and downstream-use rights have been explicitly approved. diff --git a/README.md b/README.md index 8aaeaaa..fb7ca2b 100644 --- a/README.md +++ b/README.md @@ -1,208 +1,204 @@ # Borderless Table Structuring Lab -Research infrastructure for auditable borderless-table structure recognition, -Canonical Table supervision, safe Raw MinerU refinement, and independent -Explicit and LoRA candidate routes. - -This repository is designed as the long-lived project home. The initial -revision contains the data-engineering and safety-integration layers only. It -does **not** contain model weights, training payloads, terminal benchmark pages, -Customer50 artifacts, or per-sample terminal predictions. - -## Table of contents - -- [Research objective](#research-objective) -- [Current repository scope](#current-repository-scope) -- [Repository layout](#repository-layout) -- [System design](#system-design) -- [Data strategy](#data-strategy) -- [Installation](#installation) -- [Tests](#tests) -- [Reproducibility and evidence](#reproducibility-and-evidence) -- [Collaboration workflow](#collaboration-workflow) -- [Roadmap](#roadmap) -- [Governance and licensing](#governance-and-licensing) - -## Research objective - -The project targets table-quality improvement under the OmniDocBench document -parsing protocol while preserving the Raw MinerU document baseline. The core -engineering principle is selective, auditable table correction: - -1. Raw MinerU remains the default output. -2. The Explicit route may propose minimal topology-only corrections with Raw - OCR text frozen. -3. The LoRA route may propose one complete, table-only Canonical Table state. -4. Both routes pass through the same legality, token-preservation, geometry, - provenance, expected-gain, assembly, and exact-Raw-rollback controls. -5. Unsafe or unsupported candidates are rejected without modifying Raw. - -The target of Table TEDS above 95 is an engineering objective, not a guaranteed -unobserved result. Public benchmark-aware development and independent terminal -generalization must be reported separately. - -## Current repository scope - -Included in the first revision: - -- Canonical Table normalization and structural label utilities. -- Direct-state and order-invariant target compilation. -- Candidate-integrity checks. -- Shared fail-closed validation and deterministic Raw rollback. -- Explicit topology-only and LoRA complete-table candidate interfaces. -- Synthetic unit fixtures and regression tests. -- Canonical record schema. -- English Evidence Cards and the active execution contract. -- Dataset governance, storage, reproducibility, and collaborator handoff - documentation. - -Explicitly excluded: - -- Model implementations, adapters, checkpoints, or weights. -- Full training corpora or rendered sample payloads. -- Formal20k source records and compiled record payloads. -- Customer50 content. -- OmniDocBench pages, crops, annotations, recognized strings, coordinates, - HTML, LaTeX, page identifiers, or Gold records. -- Per-sample terminal predictions or case-selection artifacts. +Research on recovering table structure from weak or absent visual boundaries. +The repository brings together canonical table representations, controlled +data generation, explicit topology modeling, and parameter-efficient +generative adaptation in a shared experimental framework. + +## Overview + +Borderless tables rarely expose their structure through ruling lines alone. +Their latent grid must be inferred from alignment, spacing, typography, +semantic grouping, spanning cells, and document context. Small structural +errors can then propagate into reading order, cell ownership, and content +alignment. + +This lab studies the problem at three connected levels: + +- **Representation:** how to describe topology, geometry, and content without + tying the target to one arbitrary edit sequence. +- **Learning:** how explicit structural prediction and generative adaptation + behave under the same data and evaluation conditions. +- **Data:** how to construct reproducible, source-traceable corpora that expose + structural phenomena systematically rather than through incidental examples. + +OmniDocBench is used as one document-parsing evaluation protocol. The methods +and infrastructure in this repository are designed around the broader research +problem of table structure recognition. + +## Research snapshot 2026.08.12 + +The `2026.08.12` snapshot establishes the shared representation and evaluation +foundation for two independent modeling tracks. It includes Canonical Table +normalization, order-invariant topology targets, candidate-integrity checks, +table-only model interfaces, and synthetic data-free tests. Model checkpoints +and dataset payloads are maintained outside this repository. + +Project-authored releases follow calendar versioning: + +- research snapshots: `2026.08.12`, `2026.09.03`, and so on; +- same-day revisions: `2026.08.12.1`, `2026.08.12.2`, and so on; +- model artifacts: `explicit-2026.08.12` and `lora-2026.08.12`. + +External software and benchmark releases retain their original upstream names. +The complete naming convention is documented in +[CONTRIBUTING.md](CONTRIBUTING.md#calendar-versioning). + +## Research tracks + +### Explicit Layout Transformer + +The explicit track treats a table as a structured object and predicts sparse, +order-invariant topology changes. Text, geometry, and cell ownership remain +separate signals so that a structural hypothesis can be inspected and replayed. +The current repository includes the candidate representation, reversible +interface, and validation primitives needed by the model. + +See [Explicit Layout Transformer](docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md). + +### LoRA Table Model + +The generative track studies parameter-efficient adaptation for direct +Canonical Table prediction. Instead of imitating one serialized sequence of +split and merge actions, the model produces a complete table hypothesis that +can be evaluated against the same representation and metrics as the explicit +track. + +The LoRA implementation and ablation studies are maintained as an independent +research contribution and are integrated through the shared Canonical Table +interface. + +## Shared experimental foundation + +```mermaid +flowchart LR + A["Table image and document context"] --> B["Visual, OCR, and geometry evidence"] + B --> C["Canonical Table representation"] + C --> D["Explicit topology model"] + C --> E["LoRA generative model"] + D --> F["Comparable table hypothesis"] + E --> F + F --> G["Topology, location, and content evaluation"] +``` + +The common foundation provides: + +- a canonical representation of rows, columns, spans, text, and geometry; +- direct-state and order-invariant structural supervision; +- deterministic rendering and synthetic-phenomenon generation; +- token-ownership and geometry-integrity checks; +- reproducible manifests, hashes, split isolation, and evidence records; +- route-independent metrics, including GriTS and TEDS-compatible adapters. ## Repository layout ```text borderless-table-structuring-lab/ -├── .github/ -│ └── workflows/ # Continuous integration -├── dataset/ -│ └── README.md # External dataset registry and access policy +├── dataset/ # External dataset registry; no payloads ├── docs/ -│ ├── COLLABORATOR_HANDOFF.md +│ ├── methods/ # Research-track formulations │ ├── DATA_GOVERNANCE.md │ ├── DATASET_STORAGE_AND_SHARING.md │ ├── REPRODUCIBILITY.md │ └── SYNTHETIC_CORPUS_SPECIFICATION.md -├── evidence/ # Nonterminal Evidence Cards only -├── governance/ # Active execution contract and seal -├── schemas/ # Versioned record schemas -├── scripts/ # Deterministic data-layer compilers -├── src/mpr_tsr_splitmerge_v2/ -│ ├── canonical.py -│ ├── candidate_integrity.py +├── schemas/ # Calendar-versioned record schemas +├── src/borderless_table_structuring/ +│ ├── canonical.py # Canonical table normalization +│ ├── explicit.py # Public Explicit-route interface │ ├── candidate_interfaces.py +│ ├── candidate_integrity.py │ ├── labels.py │ └── safety_layer.py -├── tests/ # Synthetic and deterministic unit tests -├── CONTRIBUTING.md -└── pyproject.toml +└── tests/ # Synthetic, data-free regression tests ``` -## System design - -The active representation is the final Canonical Table state plus an -order-invariant structural difference. Historical ordered KEEP/SPLIT/MERGE -programs are replay evidence, not primary supervision, because multiple action -paths may lead to the same correct table. +The research tracks use the model names **Explicit Layout Transformer** and +**LoRA Table Model**. Project-authored identifiers use the same calendar +release as the surrounding research snapshot. -The shared safety boundary checks: +## Canonical Table representation -- Canonical legality and complete grid coverage. -- OCR token ownership and text preservation. -- Finite, non-degenerate physical geometry. -- Provenance and table-only scope. -- Frozen positive expected-gain evidence. -- Non-table page-state stability. -- Exact deterministic Raw rollback. +A record connects the rendered observation to an explicit table state: -See [the synthetic corpus specification](docs/SYNTHETIC_CORPUS_SPECIFICATION.md) -for the data contract and [the active execution contract](governance/POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md) -for the current authorization boundary. +- logical grid dimensions and cell spans; +- physical cell geometry; +- OCR tokens, confidence, and unique cell ownership; +- source, renderer, template, and content provenance; +- deterministic generation parameters and hashes; +- direct Canonical Table supervision; +- an order-invariant structural difference when a prior state is available. -## Data strategy +Ordered action programs may be retained for analysis, but they are not treated +as the unique description of a correct table. -Training payloads are intentionally stored outside this Git repository. -GitHub stores schemas, manifests, checksums, generators, validators, and -Evidence Cards. A separate private dataset repository stores authorized data -shards and is pinned by immutable revision and SHA256 manifest. +## Data methodology -Recommended dataset repository name: +The data pipeline is designed to vary structure and appearance independently. +Structural families include hierarchical headers, row and column spans, +mixed-dimensional spans, empty cells, and localized split/merge corrections. +Rendering families cover border visibility, typography, resolution, rotation, +compression, blur, background, and scanning artifacts. -```text -DearKarl/borderless-table-structuring-data -``` - -The preferred format is versioned WebDataset or Parquet shards with a small -JSONL manifest. See [dataset storage and sharing](docs/DATASET_STORAGE_AND_SHARING.md) -and [the dataset registry](dataset/README.md). +Dataset roles are assigned by document, template, content, renderer, and seed +families before rendering. Exact and near-duplicate audits operate on images, +text, normalized structure, geometry, and provenance. See the +[synthetic corpus specification](docs/SYNTHETIC_CORPUS_SPECIFICATION.md) and +[data governance guide](docs/DATA_GOVERNANCE.md). ## Installation Python 3.10 or newer is required. ```bash +git clone https://github.com/DearKarl/borderless-table-structuring-lab.git +cd borderless-table-structuring-lab python -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip python -m pip install -e '.[dev]' ``` -## Tests - -Run the complete data-engineering test suite: +Run the data-free test suite: ```bash pytest ``` -The committed tests use synthetic fixtures. They do not require terminal -benchmark data, model weights, or a GPU. - -## Reproducibility and evidence +## Reproducible research -Every data stage must freeze: +Experiments record the code revision, schema release, immutable data revision, +root manifest hash, configuration hash, random seeds, environment, metrics, +and complete failure accounting. Dataset payloads and model weights are stored +outside Git; this repository contains the code, schemas, manifests, and +documentation needed to reproduce them. -- source and permitted-use inventory; -- schema version and compiler revision; -- deterministic seeds and render parameters; -- train, development, holdout, and terminal roles; -- exact and near-duplicate isolation reports; -- generated, accepted, quarantined, and failed counts; -- complete SHA256 manifests; -- an English Evidence Card and seal. +For details, see [Reproducibility](docs/REPRODUCIBILITY.md) and +[Dataset Storage and Sharing](docs/DATASET_STORAGE_AND_SHARING.md). -Detailed requirements are documented in -[REPRODUCIBILITY.md](docs/REPRODUCIBILITY.md). +## Collaboration -## Collaboration workflow +The two model tracks share representations and evaluation but keep model code +and ablations independent. Suggested branch prefixes are: -1. Read the active contract and seal before project work. -2. Work on a dedicated branch. -3. Change one declared variable per experiment. -4. Add tests and an Evidence Card for data-affecting changes. -5. Never commit datasets, credentials, weights, or terminal artifacts. -6. Submit a pull request with source, license, isolation, and replay evidence. +- `explicit/` for explicit topology modeling; +- `lora/` for generative adaptation and ablations; +- `data/` for corpus construction and validation; +- `eval/` for route-independent metrics and analysis. -See [CONTRIBUTING.md](CONTRIBUTING.md) and -[COLLABORATOR_HANDOFF.md](docs/COLLABORATOR_HANDOFF.md). +Contributions should include tests, a concise method note, and the provenance +or experimental metadata needed to interpret the result. Large datasets, +weights, credentials, and benchmark payloads must not be committed. -## Roadmap +See [CONTRIBUTING.md](CONTRIBUTING.md) for the review workflow. -- [x] Canonical Data Layer vNext. -- [x] Shared validator and exact Raw rollback. -- [x] Independent Explicit and LoRA candidate interfaces. -- [x] Bounded correctness smoke. -- [ ] Freeze the synthetic-corpus specification and collaborator handoff. -- [ ] Implement bounded generators and pass the data smoke. -- [ ] Build a frozen shared corpus with complete overlap audits. -- [ ] Run independent bounded Explicit and LoRA candidate pilots. -- [ ] Stop at the mandatory pre-full-training discussion gate. -- [ ] Add model code only after the corresponding governance decision. +## Citation -## Governance and licensing +A project citation will be added with the first archival release. Until then, +please cite the repository URL and the immutable commit used in an experiment. -The repository is private and currently has no public redistribution license. -Project code and newly authored synthetic assets require an explicit licensing -decision before public release. Third-party datasets retain their own licenses -and must not be redistributed merely because this repository is private. +## License -OmniDocBench and Customer50 terminal contents are prohibited from the training -and development corpus. Any source with uncertain redistribution or downstream -use rights must remain external until the license review passes. +No public redistribution license has been assigned yet. Third-party datasets, +fonts, evaluators, and pretrained models remain subject to their original +licenses. Consult the repository maintainers before redistributing code or +derived assets. diff --git a/dataset/README.md b/dataset/README.md index 4702250..43b0ccb 100644 --- a/dataset/README.md +++ b/dataset/README.md @@ -11,11 +11,12 @@ service: private-hugging-face-or-approved-object-store repository: DearKarl/borderless-table-structuring-data revision: null root_manifest_sha256: null -schema_version: null +schema_release: null +research_snapshot: null roles_available: [] license_manifest_sha256: null overlap_audit_sha256: null -evidence_card_sha256: null +experiment_record_sha256: null status: not_yet_frozen ``` diff --git a/docs/COLLABORATOR_HANDOFF.md b/docs/COLLABORATOR_HANDOFF.md index 145ec22..38a3cd7 100644 --- a/docs/COLLABORATOR_HANDOFF.md +++ b/docs/COLLABORATOR_HANDOFF.md @@ -2,8 +2,8 @@ ## Goal -Build a source-traceable, category-balanced, OmniDocBench-protocol-aware -synthetic corpus without exposing terminal benchmark or Customer50 artifacts. +Build a source-traceable, category-balanced synthetic corpus for borderless +table structure recognition without exposing restricted evaluation artifacts. ## Recommended first assignment @@ -21,7 +21,7 @@ Deliverables: - replay tests for every parameter family; - a small smoke manifest with pass, quarantine, and failure reason codes; - exact and perceptual duplicate reports; -- an English Evidence Card. +- an English experiment record. ## Other work packages @@ -57,4 +57,4 @@ terminal labels, or start a full model training run. A contribution is accepted only after its source/license manifest, deterministic replay test, isolation audit, complete count accounting, English -Evidence Card, and SHA256 seal pass review. +experiment record and SHA256 manifest pass review. diff --git a/docs/DATASET_STORAGE_AND_SHARING.md b/docs/DATASET_STORAGE_AND_SHARING.md index c40e820..1c470b3 100644 --- a/docs/DATASET_STORAGE_AND_SHARING.md +++ b/docs/DATASET_STORAGE_AND_SHARING.md @@ -5,7 +5,7 @@ Use two repositories with different responsibilities: 1. `DearKarl/borderless-table-structuring-lab` stores code, schemas, - generators, manifests, checksums, tests, governance, and Evidence Cards. + generators, manifests, checksums, tests, and reproducibility records. 2. `DearKarl/borderless-table-structuring-data` stores authorized dataset payloads in a private dataset service or object store. @@ -66,10 +66,10 @@ The collaborator should initially receive the synthetic training and development roles. Holdout and terminal roles remain inaccessible to anyone performing generator or model optimization. -## Existing Formal20k-derived records +## Existing derived records -Do not upload the current compiled Formal20k records until the source-license -inventory confirms redistribution and downstream-use rights for every source. +Do not upload compiled records until the source-license inventory confirms +redistribution and downstream-use rights for every source. If redistribution is not authorized, share only: - the compiler; diff --git a/docs/DATA_GOVERNANCE.md b/docs/DATA_GOVERNANCE.md index 9ded5f4..5a41645 100644 --- a/docs/DATA_GOVERNANCE.md +++ b/docs/DATA_GOVERNANCE.md @@ -21,12 +21,12 @@ Every generated or newly sourced record must include: - document, template, content, and renderer family identifiers; - SHA256 entries for every payload and manifest record. -## Terminal exclusion +## Evaluation-data exclusion -Customer50 and OmniDocBench terminal pages, crops, annotations, strings, -coordinates, HTML, LaTeX, page metadata, identifiers, Gold records, embeddings, -and transformations are prohibited inputs. No record may be selected because -it resembles a specific terminal case. +Restricted evaluation pages, crops, annotations, strings, coordinates, HTML, +LaTeX, page metadata, identifiers, Gold records, embeddings, and transformations +are prohibited inputs. No record may be selected because it resembles a +specific evaluation case. ## License gate diff --git a/docs/REPRODUCIBILITY.md b/docs/REPRODUCIBILITY.md index 7d77df9..e635ac3 100644 --- a/docs/REPRODUCIBILITY.md +++ b/docs/REPRODUCIBILITY.md @@ -3,7 +3,7 @@ ## Immutable inputs Each run records the Git commit, dataset repository and revision, root manifest -SHA256, schema version, configuration hash, generator seeds, split roles, and +SHA256, schema release, configuration hash, generator seeds, split roles, and runtime environment. ## Non-overwriting outputs @@ -16,8 +16,8 @@ Never reuse an output directory. A run writes to a new directory containing: - `report.json`; - `quarantine.jsonl`; - `SHA256SUMS`; -- `EVIDENCE_CARD.md`; -- an Evidence Card seal. +- `EXPERIMENT_RECORD.md`; +- an immutable checksum manifest. ## Determinism @@ -28,12 +28,12 @@ the normalization procedure. ## Evaluation discipline -Development metrics may compare bounded single-variable changes against the -frozen Raw baseline. Holdout and terminal roles are excluded from generator, -threshold, checkpoint, architecture, and data selection. Any future full -training command requires a separate explicit approval at the contract gate. +Development metrics may compare bounded single-variable changes against a +frozen reference. Holdout and terminal roles are excluded from generator, +threshold, checkpoint, architecture, and data selection. -## Evidence review +## Research-record review Reviewers verify source rights, role isolation, complete failure accounting, -terminal non-use, and exact Raw rollback before accepting a stage transition. +restricted-data non-use and deterministic replay before accepting a research +result. diff --git a/docs/SYNTHETIC_CORPUS_SPECIFICATION.md b/docs/SYNTHETIC_CORPUS_SPECIFICATION.md index beaf055..e8985cf 100644 --- a/docs/SYNTHETIC_CORPUS_SPECIFICATION.md +++ b/docs/SYNTHETIC_CORPUS_SPECIFICATION.md @@ -1,6 +1,8 @@ # Synthetic Corpus Specification -Status: `DRAFT_FOR_STAGE_5_4_REVIEW` +Research snapshot: `2026.08.12` + +Status: `RESEARCH_DRAFT` ## Primary record contract @@ -18,11 +20,11 @@ Required identity fields: - `renderer_family_id`; - `generation_seed`; - `role`; -- `schema_version`. +- `schema_release`. Required provenance fields: -- generator and version; +- generator and calendar release; - source and license decision; - font sources; - render and degradation parameters; diff --git a/docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md b/docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md new file mode 100644 index 0000000..0f34d0d --- /dev/null +++ b/docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md @@ -0,0 +1,93 @@ +# Explicit Layout Transformer + +## Research question + +The Explicit Layout Transformer studies whether table structure can be +recovered as an interpretable prediction over a latent grid rather than as an +unconstrained serialization. The central hypothesis is that topology, physical +geometry, and text ownership should be modeled as related but distinct +variables. + +## Representation + +Let a table state be + +\[ +T = (R, C, \mathcal{V}, \mathcal{G}, \mathcal{O}), +\] + +where `R` and `C` are grid dimensions, `V` is the set of logical cells, `G` +contains physical geometry, and `O` maps OCR tokens to cells. Each logical cell +is represented by a half-open span + +\[ +v_i = (r_i^0, r_i^1, c_i^0, c_i^1). +\] + +The learning target is an order-invariant difference between an observed table +hypothesis and the final Canonical Table state. Equivalent sequences of merge +and split operations therefore share the same target. + +## Model interface + +The model consumes visual features together with a structured table prior and +produces: + +1. a table-level identity or edit decision; +2. row and column boundary evidence; +3. cell-span or adjacency hypotheses; +4. a partition of source cells into proposed logical cells; +5. confidence values suitable for calibrated evaluation. + +The current codebase implements the representation boundary around this model: + +- `build_explicit_topology_candidate` constructs a complete table hypothesis + from a partition of source cells; +- OCR token ownership is copied from the contributing source cells; +- physical geometry is derived from their geometric union; +- every source cell must be covered exactly once; +- `replay_explicit_to_raw` verifies the source-state binding and reconstructs + the unmodified input state; +- `select_explicit_candidate` evaluates the hypothesis using the shared + candidate-validation interface. + +The trainable transformer is intentionally kept separate from these invariants +so architecture ablations cannot silently redefine the table representation. + +## Supervision + +Primary supervision uses the final Canonical Table state and a set-valued, +order-invariant topology difference. Historical action sequences may be used +for debugging or replay analysis, but they are not assumed to be unique. + +Useful auxiliary objectives include: + +- row and column boundary evidence; +- cell adjacency and span consistency; +- edit sparsity or identity prediction; +- token-ownership preservation; +- geometry consistency; +- table-level structural validity. + +Each auxiliary objective must be reported separately from the aggregate loss +so a lower training loss cannot conceal a degraded topology or content metric. + +## Evaluation + +The Explicit track is evaluated with the same data roles and canonical output +format as the LoRA track. Core measurements include: + +- GriTS Topology, Location, and Content; +- TEDS-compatible structure and content scores; +- precision and recall for structural corrections; +- cell-count inflation and deflation; +- OCR token-preservation rate; +- canonical validity and geometry coverage; +- results stratified by header, span, border, text, and imaging phenomena. + +## Current repository status + +The Canonical Table schema, topology-candidate interface, deterministic replay, +shared validation primitives, and synthetic regression tests are available. +Model architecture and training code will be added as a separate, reviewable +research contribution together with its configuration and ablation record. diff --git a/evidence/EVIDENCE_CARD_CANONICAL_DATA_LAYER_VNEXT_V1.md b/evidence/EVIDENCE_CARD_CANONICAL_DATA_LAYER_VNEXT_V1.md deleted file mode 100644 index fc6e99c..0000000 --- a/evidence/EVIDENCE_CARD_CANONICAL_DATA_LAYER_VNEXT_V1.md +++ /dev/null @@ -1,87 +0,0 @@ -# Evidence Card: Canonical Data Layer vNext V1 - -Status: `SEALED_COMPLETE` - -Execution date: 2026-08-11 (Asia/Shanghai) - -## 1. Hypothesis - -Direct Canonical Table state and order-invariant partition differences remove arbitrary action-path supervision while preserving the verified source evidence and Canonical Gold. - -## 2. Authorization and isolation - -- Active authority: `POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V4.md`. -- Dataset role: Formal20k source data and Gold, used only to compile a new versioned derived-label layer. -- Customer50 read: `false`. -- OmniDocBench terminal page or metric read: `false`. -- Full model training started: `false`. -- Formal20k v1 modified: `false`. -- Final output directory existed before the run: `false`. - -## 3. Frozen inputs - -- Formal20k v1 records: `20,000`. -- Formal20k v1 manifest SHA256: `716ad18e1c26a58ced016dcf8424049a4081c92e39215ad7b56268b3fd43aa99`. -- Contract V4 SHA256: `9a9b8e1fb0ce7729004e897715f2dba695af7cced14bb81203822ae3f9f3a343`. -- Canonical Data Layer contract SHA256: `a0215b45230471b3093eae160ac6a4eaef86d590c90be939306fec1bd79c33cf`. -- Schema SHA256: `c5793628fb19bdac859f9123ac3c8c0110305ac0e4726088f4f3d768c00efb30`. -- Preregistration SHA256: `b8f5cb736d7cebcfcb40a7dabd817326465c304bd7f292636e132a98ca7f5067`. -- Final compiler SHA256: `534e160d5caf7d734d399247153f15e14787cfffee585c24e4befa95752ad094`. -- Final compiler-test SHA256: `86ac57db19bec4a19e55777dfa943fb970b2059600ba3de30f048333bfe85a1e`. -- Input seal V2 SHA256: `2a74abbe7c2cd3ba9065d3294f02ce4731c95743e4ff19bb0f9eedaa0978cab9`. - -## 4. Preserved pre-commit failure - -Attempt 1 stopped before final output commit because historical `ocr_gold_pointer` values index primitive-grid owners rather than Canonical Gold cells. The temporary evidence was preserved and the source manifest was not modified. - -- Failure evidence SHA256: `d30bf40e0d4157956517c6a5d7fd277945335970b6c4335079e3d6126a9c377b`. -- Corrective single change: compile Canonical cell token ownership from audited `text_edit_labels.cells[].ocr_token_indexes`. -- Final unit checks: `3/3 PASS`. - -## 5. Final compile result - -| Item | Result | -|---|---:| -| Records processed | 20,000 | -| Included records | 19,992 | -| Quarantined grid-hole records | 8 | -| Canonical Gold cells with geometry | 901,304 | -| Canonical Gold cells missing geometry | 0 | -| Historical non-unique merge-path records | 1,444 | -| Historical non-unique components | 4,175 | -| Historical non-unique Raw cells | 21,807 | -| Compile failures | 0 | - -Source counts remain FinTabNet.c `8,000`, PubTables-1M `8,000`, and TabRecSet `4,000`. - -## 6. Primary supervision change - -- Primary truth: `DIRECT_CANONICAL_TABLE_STATE`. -- Explicit target: order-invariant topology partition difference with default `KEEP` and Raw-text-preserving policy. -- LoRA target: complete table-only Canonical candidate reference with OCR-copy preference and parallel/shared-sidecar geometry. -- Historical action programs: replay evidence only, identified by SHA256. -- Grid-hole policy: `PRESERVE_AND_QUARANTINE`. - -## 7. Determinism - -The complete compilation was repeated into a second non-overwriting output directory. The following files were byte-identical across runs: - -- `canonical_targets.jsonl`. -- `replay_map.jsonl`. -- `included_manifest.jsonl`. -- `quarantined_manifest.jsonl`. - -Deterministic replay status: `PASS`. - -## 8. Final output hashes - -- Canonical targets SHA256: `e4246fee34269d75d5dc24db4596a24628667b732c8495b6a2d834543def60ee`. -- Replay map SHA256: `47fa36f5657625f58c207970c57eca72ccbcaa82b814fbc198367bf568829498`. -- Included manifest SHA256: `054283401f13bac733bacd87d0e71a7085d2f096043358347e40574c7ed352fe`. -- Quarantined manifest SHA256: `97e0325820b19a0e8348a84c79e0f7d0f891f9f5544ab8da292767dad4ea29ca`. -- Compile report SHA256: `0aee80854a839716fb35a86a6aedfffa4472e82e71c43f55ee7930d0f7640931`. -- Output SHA256SUMS SHA256: `899c3e2c8df19490009c1db53bc8946dc7cfcf85d6cc0df1643f8b4e60eb5f0e`. - -## 9. Hard-gate outcome - -All Stage 1 hard gates passed. Canonical Data Layer vNext may progress to the shared validator and deterministic Raw rollback stage. This Evidence Card does not authorize full model training. diff --git a/evidence/EVIDENCE_CARD_SHARED_SAFETY_LAYER_V1.md b/evidence/EVIDENCE_CARD_SHARED_SAFETY_LAYER_V1.md deleted file mode 100644 index e4a45eb..0000000 --- a/evidence/EVIDENCE_CARD_SHARED_SAFETY_LAYER_V1.md +++ /dev/null @@ -1,65 +0,0 @@ -# Evidence Card: Shared Safety Layer V1 - -Status: `PASS_SEALED_FOR_STAGE3` - -Authority: `POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V4.md` - -## Hypothesis - -A shared fail-closed validator can preserve Raw MinerU exactly when a table-only candidate is missing, invalid, or lacks positive preregistered expected-gain evidence, while accepting only Canonical-valid candidates and freezing all non-table page state. - -## Scope - -This is correctness evidence only. It does not measure model quality and does not authorize training. The fixture set is the first 32 records in the frozen order of the compliant 169-sample nonterminal development manifest. Fixture selection read neither development Gold nor metrics. Customer50 and OmniDocBench contents were not visible. - -## Frozen interfaces - -- Canonical interval, declared-shape, unique-cell, non-overlap, and complete-grid validation. -- Physical geometry presence, finiteness, and non-degeneracy validation. -- OCR-token ownership coverage and uniqueness protection when token ownership is present. -- Frozen-Raw or OCR-grounded text protection selected by an explicit policy. -- Required sample, producer, producer-version, purpose, image-hash, and terminal-visibility provenance. -- Caller-supplied nonterminal expected-gain policy; this module does not estimate or calibrate a threshold. -- Exact Raw rollback and identity `PASS_THROUGH`. -- Table-only page assembly with non-table blocks hash-frozen and full-page candidates rejected. - -## Preregistered fixture result - -| Check | Result | -|---|---:| -| Compliant nonterminal fixtures | 32/32 | -| Raw baseline validator PASS | 32/32 | -| Identity pass-through | 32/32 | -| Identity output SHA equals Raw SHA | 32/32 | -| Deterministic corruptions | 160 | -| Corruptions rolled back | 160/160 | -| Rollback output SHA equals Raw SHA | 160/160 | -| Terminal benchmark visibility | false | -| Gold or metric visibility during fixture selection | false | - -The five corruption classes were grid overlap, grid hole, geometry removal, text mutation, and terminal-visibility provenance violation. - -## Unit and integration tests - -Twenty-two tests passed across the new safety-layer tests and the existing atomic executor rollback tests. They cover identity pass-through, overlap, token loss, text hallucination, geometry, provenance, missing gain, insufficient gain, valid positive-gain acceptance, table-only assembly, full-page candidate rejection, and pre-existing executor rollback behavior. - -## Preserved failed attempt - -`fixture_run_v1` is retained unchanged. Its behavioral checks all passed, but aggregate status was falsely reported as FAIL because two required non-visibility facts were represented as false-valued check entries before applying `all(checks.values())`. `FAILED_FIXTURE_RUN_V1.json` records the cause. The corrected non-overwrite run is `fixture_run_v2`. - -## Hard-gate decision - -All Stage 2 hard gates pass. Stage 3 candidate-interface implementation may begin. No smoke, bounded performance experiment, or training result is claimed here. - -## SHA256 record - -- Active V4 contract: `9a9b8e1fb0ce7729004e897715f2dba695af7cced14bb81203822ae3f9f3a343` -- Canonical Data Layer vNext Evidence Card seal: `1d390e59875a28cc0b9b4e4b58ca64d14e4b97ce26a3f1d98a3f420790fe59bf` -- Frozen compliant fixture manifest: `c8ce22b7cbc3964ddc1de895842110db7d59123274772b8458678f43539ff103` -- Shared safety implementation: `6cd41d5a01d5942f6ce7b82f06c17d0b4302b02836242117ab295698d71d2e90` -- Fixture runner: `94a50e323be94c168fb11fdfebe7d45540d7a2adf3c921490d7d6a8d6e42b433` -- Safety tests: `47867a5fcfe3c36c5f90ac76c76920299c0e96c056fad0dc2ab88f156a1d233c` -- Stage 2 contract: `d641fd3db15b64b99c2d65c02324db24c5e0f401ca2413b2d39810211ccf0957` -- Stage 2 preregistration: `1079efd030141e1d5a2d8bf6e1a3784a30f8865fc89c96bae76dc4de5dcd4c2f` -- Corrected fixture report: `5948ba25645cb669356ec3349828178dc27e900e47ffc8f5ca32b273f9481683` -- Corrected fixture SHA manifest: `82396c3d54e0d66ab9a3de0988e445342e13952dbcb61b23883ebc6f3c58a635` diff --git a/evidence/EVIDENCE_CARD_STAGE3_CANDIDATE_INTERFACES_V1.md b/evidence/EVIDENCE_CARD_STAGE3_CANDIDATE_INTERFACES_V1.md deleted file mode 100644 index f489aa5..0000000 --- a/evidence/EVIDENCE_CARD_STAGE3_CANDIDATE_INTERFACES_V1.md +++ /dev/null @@ -1,121 +0,0 @@ -# Evidence Card: Stage 3 Candidate Interfaces V1 - -Status: `PASS_SEALED_FOR_STAGE4` - -Authority: `POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V4.md` - -## Objective - -Verify that the Explicit Layout Transformer and LoRA Table Model expose two -independent table-only candidate interfaces behind the sealed Stage 2 shared -safety layer. This stage verifies deterministic interface correctness only. It -does not train a model, select a configuration, inspect a terminal benchmark, -or make a performance claim. - -## Frozen hypothesis and single variable - -Hypothesis: independent table-only interfaces can enforce default `KEEP`, OCR -token preservation, reversible Explicit topology replay, complete Canonical -LoRA output, shared validation, and deterministic Raw rollback without terminal -data visibility. - -Single implementation variable: `candidate_interface_contract`. - -Preregistration SHA256: -`8f41e6e9577529ef75435d21af04d50b208f477127c999134f92c408b9503d87` - -## Inputs and isolation - -- The interface-specific checks use one deterministic synthetic table fixture. -- The shared-safety regression uses the first 32 records in the frozen order of - the compliant 169-sample nonterminal development manifest. -- Frozen development manifest SHA256: - `c8ce22b7cbc3964ddc1de895842110db7d59123274772b8458678f43539ff103`. -- Customer50 and OmniDocBench inputs, predictions, cases, and metrics were not - opened or used. -- V6 and Formal20k v1 were not executed or modified. -- No GPU and no model runtime were required. - -## Implementation boundary - -Explicit interface: - -- `None` is the default `KEEP` proposal and yields exact Raw pass-through. -- A changed proposal must partition every Raw cell exactly once. -- Text is reconstructed only from owned OCR tokens. -- Geometry is the deterministic union of source Raw boxes. -- Replay metadata binds the candidate to the exact Raw state hash. - -LoRA interface: - -- Output is one complete Canonical Table candidate. -- Full-page `blocks` or `page` output is rejected. -- OCR-grounding, geometry, coverage, provenance, and Canonical validity remain - enforced by the shared safety layer. -- The interface does not define or select any LoRA training configuration. - -## Preregistered results - -All eight assertions passed: - -1. Default `KEEP` is exact Raw pass-through. -2. Changed Explicit topology preserves complete OCR token coverage. -3. Explicit replay restores the exact Raw hash. -4. Incomplete Explicit source coverage is rejected. -5. LoRA output remains table-only. -6. Hallucinated LoRA text rolls back exactly to Raw. -7. Full-page output is rejected at the interface boundary. -8. The Stage 2 shared-safety regression remains fully passing. - -The Stage 2 regression retained: - -- `32/32` valid baselines. -- `32/32` exact identity pass-throughs. -- `160/160` deterministic corruption rollbacks. -- `160/160` exact rollback hashes. - -Stage 3 report SHA256: -`54bb525b0d7a383515c251b9752a4821cefde7b3f82182eaed662a41f5368e97` - -Stage 3 SHA manifest SHA256: -`e50f441d7a98fc741e002a72a9ed88e94cce3426b336ef3bbaea170925e7616b` - -## Code and contract hashes - -- Candidate interfaces: - `e58b17893b29583eb6a8293413fdf034bf1f591a2c73a58245465179d17a825f` -- Shared safety layer: - `6cd41d5a01d5942f6ce7b82f06c17d0b4302b02836242117ab295698d71d2e90` -- Stage 3 deterministic check runner: - `b72b68482f11624d8231d97c7cd04cda22bd8539d50d00808f52a411076a8d77` -- Stage 3 contract: - `79d677f8ca82bcf9b33b0de8da274bfdc977e13e9bc732b11a40017321e46a9f` -- Stage 2 regression report: - `5948ba25645cb669356ec3349828178dc27e900e47ffc8f5ca32b273f9481683` - -## Interpretation - -Stage 3 establishes that both route boundaries now enforce the intended safety -contracts before any model candidate can replace Raw MinerU. The Explicit route -can express reversible topology changes while freezing OCR-grounded text. The -LoRA route can submit a complete Canonical Table candidate but cannot rewrite a -page or pass hallucinated text through the shared validator. - -These checks do not establish model quality or expected metric gain. They only -make the next correctness-smoke stage auditable and safe. - -## Hard-stop audit - -- Terminal input or metric visibility: `FALSE`. -- Full model training command: `FALSE`. -- V6 modification or execution: `FALSE`. -- Formal20k v1 overwrite: `FALSE`. -- Route hybridization: `FALSE`. -- Assertion failure: `FALSE`. -- Performance claim authorized: `FALSE`. - -## Decision - -Stage 3 is complete and sealed. The next authorized activity is Stage 4: -preregistered correctness smokes. Stage 4 remains correctness-only, and all -work must still stop before any full model training command. diff --git a/evidence/EVIDENCE_CARD_STAGE4_CORRECTNESS_SMOKE_V2.md b/evidence/EVIDENCE_CARD_STAGE4_CORRECTNESS_SMOKE_V2.md deleted file mode 100644 index 179f015..0000000 --- a/evidence/EVIDENCE_CARD_STAGE4_CORRECTNESS_SMOKE_V2.md +++ /dev/null @@ -1,50 +0,0 @@ -# Evidence Card: Stage 4 Correctness Smoke V2 - -Status: `PASS_SEALED_FOR_STAGE5` - -Authority: `POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V5.md` - -## Objective - -Verify bounded end-to-end correctness of the independent Explicit and LoRA -table-only interfaces, the shared validator, exact Raw rollback, and table-only -page assembly on eight frozen nonterminal records. - -## Isolation and hard caps - -- Records: `8/8`, first records in frozen manifest order. -- Customer50 or OmniDocBench content visible: `false`. -- Development Gold or performance metrics visible: `false`. -- Training steps: `0`. -- GPU seconds: `0`. -- Full-page rewrites: `0`. - -## Results - -All ten preregistered assertions passed. Explicit default KEEP, valid OCR-copy -candidate acceptance, and exact replay passed for `8/8`. Valid complete LoRA -table candidates passed for `8/8`. Hallucinated text and missing geometry -rolled back exactly to Raw for `8/8`. Non-table page state remained byte-stable -for `8/8`. Full-page output was rejected at the interface boundary. - -## Interpretation - -Stage 4 establishes correctness only. It does not establish model quality, -expected metric gain, or a surviving training configuration. A pass authorizes -only Stage 5 bounded nonterminal single-variable development experiments. - -## Hashes - -- Frozen manifest SHA256: `c8ce22b7cbc3964ddc1de895842110db7d59123274772b8458678f43539ff103`. -- Report SHA256: `2fd1a1c3d182f9f97df0a249dc1a0523045a09b92274aedf39cdca9c35fbf576`. -- SHA manifest SHA256: `50d1766c95036f5b6f9e519a31d97c21da3287687404a4a99ab8e03bb94066df`. -- Runner SHA256: `8776b0609a06dea7a6f4e03e03c42dd40cc74500c03c7afd1cad63f048d6dc1f`. -- Candidate interface SHA256: `e58b17893b29583eb6a8293413fdf034bf1f591a2c73a58245465179d17a825f`. -- Shared safety layer SHA256: `6cd41d5a01d5942f6ce7b82f06c17d0b4302b02836242117ab295698d71d2e90`. -- Preregistration SHA256: `a9c12c8d736815f2d2efecd6a9e2dbd57e18b51899e0f71ee280c3dd2f784512`. - -## Decision - -Stage 4 is complete and sealed. Stage 5 may begin under V5. Full model training -remains prohibited until the mandatory pre-full-training discussion gate and -new explicit user approval. diff --git a/governance/POST_OMNIDOCBENCH_CONTRACT_V6_SEAL.json b/governance/POST_OMNIDOCBENCH_CONTRACT_V6_SEAL.json deleted file mode 100644 index 28df298..0000000 --- a/governance/POST_OMNIDOCBENCH_CONTRACT_V6_SEAL.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "schema_version": "mpr-tsr/post-omnidocbench-execution-contract-v6-seal", - "status": "FROZEN_ACTIVE_BENCHMARK_AWARE_SYNTHETIC_DATA_ENGINEERING", - "effective_at": "2026-08-12T00:00:00+08:00", - "language": "English", - "communication_language": "Chinese unless the user requests otherwise", - "contract_v5": { - "path": "POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V5.md", - "sha256": "651d41371473c5e4959be5da89541b27014de7d4e6d0109d5c96b5862c504279", - "preserved": true - }, - "contract_v6": { - "path": "POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md", - "sha256": "9972d93a519f07b8cf3baea1529cc5fbf67deb1fc96828c743c25a7472c0873a", - "active": true, - "mandatory_full_read_before_every_action": true - }, - "active_target": "OMNIDOCBENCH_PROTOCOL_TABLE_TEDS_TOWARD_95_PLUS_WITH_RAW_NON_REGRESSION", - "benchmark_aware_category_engineering": true, - "terminal_page_or_gold_derivation_authorized": false, - "synthetic_omni_like_corpus_authorized": true, - "collaborator_handoff_authorized": true, - "raw_default": true, - "explicit_route": "TOPOLOGY_ONLY_DEFAULT_KEEP_RAW_TEXT_FROZEN", - "lora_route": "COMPLETE_CANONICAL_TABLE_CANDIDATE_TABLE_ONLY", - "primary_supervision": "FINAL_CANONICAL_TABLE_STATE_AND_ORDER_INVARIANT_STRUCTURAL_DIFFERENCE", - "completed_state": "STAGE4_AND_STAGE5_EXPERIMENTS_01_TO_03_SEALED", - "next_authorized_stage": "STAGE5_4_SYNTHETIC_DATA_SPECIFICATION_AND_COLLABORATOR_HANDOFF", - "hard_stop_before": "ANY_FULL_DATASET_MODEL_TRAINING_COMMAND" -} diff --git a/governance/POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md b/governance/POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md deleted file mode 100644 index 93ceef6..0000000 --- a/governance/POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md +++ /dev/null @@ -1,513 +0,0 @@ -# Post-OmniDocBench Benchmark-Aware Data Engineering Execution Contract and Project Handoff V6 - -Status: `FROZEN_ACTIVE_BENCHMARK_AWARE_SYNTHETIC_DATA_ENGINEERING` - -Effective date: 2026-08-12 (Asia/Shanghai) - -Scope: OmniDocBench-protocol-targeted table-quality improvement, independent -synthetic and source-traceable Omni-like data construction, Canonical Data -Layer vNext, Raw MinerU non-regression, Explicit Layout Transformer, LoRA -Table Model, collaborator handoff, bounded nonterminal evidence, and a future -untouched terminal acceptance protocol. - -## 1. Mandatory full-read rule before every action - -This V6 file is the sole active execution contract and project handoff. - -Before every new task, subtask, automation turn, code change, optimization, -data operation, diagnostic, experiment, smoke test, inference run, evaluation, -report edit, or training request, the executor must: - -1. Locate this exact V6 file and its V6 seal. -2. Read both files in full from beginning to end. -3. Verify the V6 SHA256 against the seal. -4. Confirm that the requested action is authorized by V6. -5. Confirm the benchmark version, data role, isolation status, and license of - every input before development work. -6. Confirm that every output uses a new non-overwriting path. -7. Confirm that preregistration, metrics, hard stops, and Evidence Card rules - exist before mutation. -8. Stop and ask the user before any action that exceeds V6. - -The only actions permitted before this read are locating and reading V6 and -its seal. Reading an earlier contract is not a substitute for reading V6. - -## 2. Project language and communication - -- All new or actively revised project-authored files, filenames, - documentation, code comments, manifests, Evidence Cards, and contracts must - be English. -- Historical sealed artifacts remain unchanged. -- User-facing discussion and status synchronization remain Chinese unless the - user asks otherwise. -- The Desktop weekly report remains frozen until the user explicitly requests - another update. - -## 3. Authority and preserved history - -V6 supersedes V5 after the user's explicit approval of benchmark-aware, -category-level, synthetic-data optimization. V1 through V5 and all sealed -terminal artifacts remain immutable. - -- V1 SHA256: - `35dbd02707fbcf9f53465cd57dd675ec5dfd1d48f622d8e139fd797c162e8bfc` -- V2 SHA256: - `b95b8e68ce6b35e6156e6fc0b79e00667962cc631be2e5f7d36f278c5bd08e87` -- V3 SHA256: - `7b745e2c1400fe38292471bb93a359112898ac87d6f18eab6685a16f0f6f4519` -- V4 SHA256: - `9a9b8e1fb0ce7729004e897715f2dba695af7cced14bb81203822ae3f9f3a343` -- V5 SHA256: - `651d41371473c5e4959be5da89541b27014de7d4e6d0109d5c96b5862c504279` - -Where V6 conflicts with an earlier contract, V6 controls. The permanently -closed Formal20k V6 model is distinct from this execution-contract version and -remains immutable. - -## 4. Active engineering objective - -The sole active engineering north star is strong performance under the -OmniDocBench full-page document-parsing protocol, with the immediate table -objective of raising Table TEDS toward and above `95` while preserving the Raw -MinerU document baseline. - -The frozen v1.6 reference boundary is: - -| System | Overall | Table TEDS | Structure TEDS | -|---|---:|---:|---:| -| Raw MinerU | 95.60795 | 93.0863 | 95.7096 | -| Frozen V6 Full-Edit | 86.91410 | 67.0082 | 77.2006 | - -The intended gain is therefore not a broad rewrite of already-correct pages. -It is selective recovery of table errors without reducing Overall, structure, -text, formula, reading order, geometry, or deterministic rollback quality. - -`95+` is a target, not a guaranteed unobserved measurement. V6 guarantees an -auditable process, exact Raw rollback, preregistered selection, and a separate -generalization check. - -## 5. Benchmark governance finding and allowed interpretation - -The official OmniDocBench project publicly distributes the annotated dataset, -evaluation code, page and table attribute schema, local result-generation -tools, and public benchmark tables. The reviewed official materials do not -state a hidden-server submission limit or an explicit prohibition against -using public task attributes to design training data. - -Accordingly, V6 authorizes `benchmark-aware category-level engineering`: - -- Optimize for the public OmniDocBench task definition, output format, - evaluator behavior, and published attribute taxonomy. -- Use already sealed aggregate scores to define the product boundary. -- Use independently confirmed error categories to decide which generic table - phenomena require coverage. -- Generate new source-traceable tables that express those phenomena without - copying the benchmark pages or answers. - -This does not authorize claiming that a benchmark-aware result is an untouched -blind estimate. Reports must distinguish public benchmark performance from an -independent terminal generalization result. - -Official and primary references: - -- OmniDocBench repository: https://github.com/opendatalab/OmniDocBench -- OmniDocBench CVPR 2025 paper: https://arxiv.org/abs/2412.07626 -- Official dataset card and license: https://huggingface.co/datasets/opendatalab/OmniDocBench -- PureDocBench contamination and traceability analysis: - https://arxiv.org/abs/2605.07492 - -## 6. Benchmark version policy - -- The existing sealed comparison remains `OmniDocBench v1.6 Full` because its - evaluator, manifest, predictions, and baseline are already frozen. -- Scores from different OmniDocBench versions are not directly comparable. -- The current official repository may advance beyond v1.6. Migration to a - newer leaderboard version requires a separately frozen evaluator commit, - dataset manifest, Raw baseline, and one-shot protocol before model - prediction. -- No experiment may silently mix v1.6 data, metrics, or matching behavior with - another release. - -## 7. Immutable terminal and frozen-artifact boundary - -- Formal20k training remains closed at `epoch19 / global_step23826`. -- Frozen checkpoint SHA256: - `1130bc6dcf77f33820a173e9cfcad463999ee9da7a0339b49d65dd45aad53877`. -- Frozen config SHA256: - `75449ad6394afc5d0072b6e6a61eef65e4ce1ec7931f4ee6dc3c5244ec1fd4a1`. -- The frozen Formal20k V6 model may not be modified, continued, retrained, - reevaluated, or used as a successor foundation. -- Customer50 remains sealed and may not be regenerated, selected, tuned, or - reevaluated. -- Existing OmniDocBench v1.6 predictions, Gold isolation history, official - results, and hashes remain immutable. -- Existing OmniDocBench pages, crops, OCR strings, coordinates, HTML, LaTeX, - page IDs, filenames, and Gold records may not be copied into training or - development data. -- Existing OmniDocBench pages may not be transformed, paraphrased, rendered, - cropped, degraded, augmented, or otherwise used to create near-duplicate - training examples. -- No sample-specific rule keyed by a benchmark filename, page ID, text string, - coordinate pattern, or Gold fragment is permitted. -- The current official evaluation may not be rerun during data generation or - model-selection loops under V6. - -The sealed official results/SHA256SUMS SHA256 remains: -`cfe306a0ccdd4c67db3473ae5c9c3fc9fbb7f1ef88db44714d4ab7cc22202752`. - -## 8. Confirmed attribution and data verdict - -The four-level diagnostic proved that Raw MinerU, Adapter Identity, and V6 -No-Edit are identities on shared measures. Degradation begins at Full-Edit and -is associated with over-editing, excessive or incorrect split/merge, token -reassignment, cell inflation, and unstable action supervision. - -Formal20k is not globally corrupt: - -- `20,000/20,000` Canonical Gold records are valid and replayable. -- Eight localized Raw grid-hole records are quarantined. -- `1,444/20,000` records expose multiple equivalent merge paths. -- Direct final Canonical state and order-invariant structural differences are - the active supervision representation. - -## 9. Source-traceable Omni-like synthetic data policy - -V6 authorizes construction of a new shared training corpus for the Explicit -and LoRA routes. Every record must be newly generated or come from an approved -nonterminal source with compatible research and downstream-use rights. - -### 9.1 Required generated record - -Each synthetic or newly sourced record must contain: - -- Source or generator provenance and license. -- Template-family ID and deterministic generation seed. -- Render parameters, font sources, resolution, and degradation parameters. -- Source table content and direct Canonical Table Gold. -- OCR tokens, token ownership, confidence, and physical geometry. -- Raw-like prior state produced without terminal data. -- Order-invariant structural differences from Raw-like prior to Gold. -- Difficulty and phenomenon tags. -- Document-cluster, template-family, content-family, and renderer-family IDs. -- Complete SHA256 manifest entries. - -### 9.2 Authorized target phenomena - -The coverage plan may target these generic categories because they follow the -public task taxonomy and independently confirmed development risks: - -- Multi-level and irregular headers. -- Row spans, column spans, and mixed two-dimensional spans. -- Equivalent merge paths to one final table state. -- Split-only, merge-only, and joint split-plus-merge corrections. -- Weak, missing, partial, or low-contrast borders. -- Empty cells and visually ambiguous near-empty cells. -- Dense small text, long text, multiline text, and mixed scripts. -- Formula and ordinary-text mixtures within tables. -- Rotated tables, color backgrounds, scanning noise, blur, compression, and - resolution variation. -- Tables where Raw-like structure is already correct and `KEEP` is the only - safe action. -- Tables with localized topology errors that admit a minimal reversible edit. - -### 9.3 Prohibited derivation - -The generator may not ingest OmniDocBench or Customer50 terminal images, -crops, Gold, recognized strings, HTML, LaTeX, coordinates, page metadata, or -case identifiers. It may not use embeddings, retrieval, style transfer, or -image-to-image synthesis seeded by terminal pages. - -### 9.4 Split and near-duplicate isolation - -- Split by document cluster, template family, content family, renderer family, - and generation seed before rendering. -- No template family may appear in more than one of train, development, - holdout, or terminal roles. -- Run exact image, perceptual image, normalized structure, normalized text, - document-source, and geometry-signature overlap audits. -- Preserve all overlap reports. Any unresolved overlap is a hard stop. -- Do not select synthetic examples because they resemble a specific terminal - page. Select only from a preregistered category and parameter distribution. - -## 10. Shared model strategy - -### 10.1 Raw MinerU default - -Raw MinerU remains the default output and hard non-regression boundary. A -candidate replaces Raw only after passing canonical legality, complete grid -coverage, OCR token preservation, complete geometry, provenance, table-only -scope, and frozen positive expected-gain requirements. Every failure returns -an exact Raw copy. - -### 10.2 Explicit Layout Transformer - -- Safety-control route with default `KEEP`. -- Topology-only editing with Raw text frozen. -- Minimal reversible structural differences bound to the Raw state hash. -- Primary targets: missing corrections, excessive splits, incorrect merges, - and cell inflation. - -### 10.3 LoRA Table Model - -- Capability-challenger route producing one complete Canonical Table - candidate. -- Direct-state supervision rather than an arbitrary action serialization. -- Table-only output; no full-page rewriting. -- Uses the same validator, assembler, expected-gain interface, and exact Raw - rollback as Explicit. - -### 10.4 Route independence - -The routes share documents, Canonical Gold, splits, OCR, geometry, evaluator, -validator, assembler, statistics, and hard stops. They remain independent -until each has sealed evidence. Hybridization is not authorized. - -## 11. Collaborator work packages - -A collaborator may contribute to the shared corpus without receiving terminal -OmniDocBench or Customer50 artifacts. The preferred division is: - -### Work Package A: benchmark-aware coverage specification - -- Convert the public OmniDocBench attribute taxonomy and the approved generic - error taxonomy into an English coverage matrix. -- Define parameter ranges and minimum counts for each phenomenon. -- Deliver a schema, examples made from newly authored content, and a license - inventory. - -### Work Package B: structural template generator - -- Implement generators for multi-level headers, complex spans, split/merge - combinations, empty cells, and minimal-correction cases. -- Emit direct Canonical Gold, geometry, ownership, and deterministic seeds. -- Include substantial `KEEP` coverage so editing is not the default prior. - -### Work Package C: rendering and degradation generator - -- Create independent fonts, border conditions, backgrounds, rotations, - resolutions, blur, compression, scanning noise, and color variations. -- Preserve exact source-to-render provenance and deterministic replay. - -### Work Package D: validator and replay QA - -- Independently validate canonical legality, grid coverage, token ownership, - geometry completeness, deterministic rendering, and Gold recompilation. -- Produce machine-readable failure accounting rather than manually removing - failed samples. - -### Work Package E: isolation and duplicate audit - -- Implement document, template, content, image, structure, text, geometry, and - near-duplicate checks across all roles. -- Produce a signed manifest and quarantine list. - -The collaborator must not inspect local terminal artifacts, select Omni or -Customer50 bad cases, tune official metrics, modify the official evaluator, or -start a full model training run. Contributions enter the project only after an -English Evidence Card, source/license manifest, reproducibility test, and -isolation audit pass. - -## 12. Completed evidence state - -### Stage 1: Canonical Data Layer vNext - -Status: `SEALED_COMPLETE`. - -- `19,992` records included and eight grid-hole records quarantined. -- Direct-state and order-invariant supervision generated and replayed. - -### Stage 2: shared validator and exact Raw rollback - -Status: `SEALED_COMPLETE`. - -- `32/32` identity pass-throughs passed. -- `160/160` injected corruptions rolled back exactly. - -### Stage 3: candidate interfaces - -Status: `SEALED_COMPLETE`. - -- Explicit and LoRA table-only interfaces passed all eight assertions. -- Evidence Card SHA256: - `d399d536012661a4fe74c729f7208c49c140f83b1da600271e1840ba80eb8e61`. - -### Stage 4: correctness smoke - -Status: `PASS_SEALED_FOR_STAGE5`. - -- Eight nonterminal records and ten assertions passed. -- No training or performance claim occurred. -- Evidence Card file SHA256: - `2b823c2312a39f2a11e3f497e2d80e20b47aadb5fd09a209993bacd09c6b26b3`. - -### Stage 5 experiments 01 through 03 - -Status: `PASS_SEALED_FOR_SYNTHETIC_DATA_AND_CANDIDATE_PILOTS`. - -- Experiment 01 confirmed stable direct-state and order-invariant supervision. -- Experiment 02 rejected the tested fixed-topology Raw-text projection and - unconditional Full-Edit replacement. -- Experiment 03 proved `169/169` deterministic Raw rollbacks and rejected all - ineligible frozen Full-Edit candidates. -- No positive trained Explicit or LoRA capability candidate exists yet. - -## 13. Authorized next sequence - -### Stage 5.4: synthetic-data specification and collaborator handoff - -Freeze the record schema, category coverage matrix, generator parameter -distributions, source/license policy, split roles, overlap audits, acceptance -tests, and collaborator work packages before bulk generation. - -### Stage 5.5: bounded generator implementation and data smoke - -- Generate a small preregistered sample from every authorized category. -- Verify deterministic replay, direct Gold recompilation, token ownership, - physical geometry, category coverage, and all overlap checks. -- Selected visual examples are QA evidence only, not model-performance - evidence. - -### Stage 5.6: frozen shared-corpus build - -- Build new non-overwriting train, development, and holdout roles only after a - sealed data-smoke pass. -- Report all requested, generated, passed, quarantined, and failed counts. -- Freeze the complete corpus manifest before model work. - -### Stage 5.7: independent bounded candidate pilots - -- Run one preregistered bounded Explicit pilot and one independent LoRA pilot. -- Use the same frozen data roles, Raw baseline, evaluator, validator, and hard - stops. -- One variable per experiment. No route mixing. -- A pilot that materially regresses Raw, text, topology, geometry, canonical - legality, assembly, or rollback determinism is rejected. - -### Stage 6: mandatory pre-full-training discussion gate - -Stop before any full-dataset model training command. Present the synthetic-data -Evidence Cards, overlap audits, bounded candidate evidence, estimated compute, -and at most one frozen full-training configuration per surviving route. Full -training requires new explicit user approval after this discussion. - -## 14. Metrics and hard rejection rules - -Every bounded development experiment reports, where valid: - -- Official-version Table TEDS and Structure TEDS adapters when frozen. -- GriTS Topology, Location, and Content. -- Precision and recall components. -- Coverage and OCR text-preservation rate. -- Physical geometry coverage. -- Cell inflation and deflation. -- Canonical validity and Gold recompilation. -- Candidate acceptance and reason-code distribution. -- Deterministic Raw rollback and full-page assembly stability. -- Document-cluster bootstrap confidence intervals. - -Reject a route or configuration if it materially regresses Raw, loses OCR -text, creates illegal topology, has incomplete geometry, changes non-table page -state, inflates cells without positive evidence, or fails deterministic -rollback. Public terminal scores may not calibrate a pilot threshold. - -## 15. Evidence, licensing, and non-overwrite requirements - -Every dataset stage or experiment requires: - -- Written hypothesis and single variable. -- Frozen inputs, sample roles, generation seeds, and SHA256 hashes. -- Source, license, and permitted-use manifest. -- Preregistered metrics, acceptance rules, and hard stops. -- New non-overwriting output paths. -- Complete coverage and failure accounting. -- Exact and near-duplicate isolation reports. -- Explicit confirmation that terminal contents were not used. -- English Evidence Card and seal before the next stage. - -OmniDocBench is distributed for research use under a noncommercial dataset -license. Its pages and annotations must not enter a customer-commercial -training corpus. Newly generated corpus rights must be documented separately. - -## 16. Current authorization boundary - -Authorized now: - -- Stage 5.4 synthetic-data specification and collaborator handoff. -- Stage 5.5 bounded generator implementation and data smoke after the Stage - 5.4 specification is sealed. -- Stage 5.6 frozen shared-corpus build after a sealed data-smoke pass. -- Stage 5.7 bounded Explicit and LoRA candidate pilots after the corpus is - sealed. -- Read-only operational and GPU-ownership monitoring. -- English Evidence Cards, manifests, audits, and seals required by these - stages. - -Not authorized: - -- Any full-dataset Explicit or LoRA model training. -- Automatic escalation to full training. -- Formal20k V6 modification, continuation, retraining, reevaluation, or - successor construction. -- Customer50 use, regeneration, selection, tuning, or reevaluation. -- Copying, transforming, augmenting, retrieving from, or training on current - OmniDocBench pages or Gold. -- Repeated current OmniDocBench evaluation for threshold, checkpoint, - architecture, prompt, rank, schedule, or data selection. -- Formal20k v1 in-place modification. -- Route hybridization. -- Desktop weekly-report editing without a new user request. -- LaunchAgent/watch modification or termination. -- EasyConnect, FTQ, or other local-network-manager restart or modification. -- OTP triggering or interference with another user's process or allocation. - -## 17. Operational safeguards - -Before remote work, perform only the read-only checks specified by the active -local environment and V6: VPN/container and routes, NJU resolution, proxy, -BatchMode SSH, caffeinate, LaunchAgent/watch singleton state, duplicate -Formal20k evidence, GPU ownership, frozen hashes, and output integrity. - -A GPU is not empty if it has any process, at least 512 MiB memory use, or at -least 5 percent utilization. Do not stop or displace another user's process. - -## 18. Active artifact index - -Base directory: -`table_without_line_benchmark/mpr_tsr_splitmerge_v2/results/formal20k_posttraining_eval_v1/` - -- Active Contract V6: `POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V6.md`. -- Contract V6 seal: `POST_OMNIDOCBENCH_CONTRACT_V6_SEAL.json`. -- Preserved Contract V5: `POST_OMNIDOCBENCH_EXECUTION_CONTRACT_V5.md`. -- Canonical Data Layer: `canonical_data_layer_vnext_v1/`. -- Shared safety layer: `shared_safety_layer_v1/`. -- Candidate interfaces: `candidate_interfaces_v1/`. -- Correctness smoke: `correctness_smoke_v1/`. -- Bounded development experiments: `bounded_development_experiments_v1/`. -- Frozen nonterminal attribution pool: `post_omnidoc_attribution_v1/`. -- New synthetic-data artifacts must use a new directory whose name and schema - are frozen during Stage 5.4. - -## 19. Resume checklist - -After reading V6 in full, confirm before action: - -1. V6 hash matches its seal. -2. Frozen checkpoint, config, and terminal hashes remain unchanged. -3. The requested work is inside Section 16. -4. Every input has a declared role, source, license, and isolation status. -5. No OmniDocBench or Customer50 terminal content enters generated data. -6. Output paths are new and non-overwriting. -7. Preregistration and hard stops exist before mutation. -8. No full model training command is started before Stage 6 approval. -9. Remote access and GPU use satisfy Section 17. - -If any check fails, stop before mutation and report the issue. - -## 20. Active execution instruction - -Prioritize targeted OmniDocBench-protocol table improvement through a new, -source-traceable, category-balanced Omni-like corpus. Preserve Raw MinerU by -default. Complete the synthetic-data specification, bounded generator smoke, -frozen shared-corpus build, and independent Explicit and LoRA candidate pilots -in order. Reject every unsafe or regressing configuration. Stop before full -training and discuss the frozen survivors with the user. diff --git a/pyproject.toml b/pyproject.toml index 53297e8..e61bec1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "borderless-table-structuring-lab" -version = "0.1.0" -description = "Auditable data engineering and safe candidate integration for borderless table structuring." +version = "2026.8.12" +description = "Research on canonical representations, explicit topology modeling, and generative adaptation for borderless table structure recognition." readme = "README.md" requires-python = ">=3.10" authors = [ diff --git a/schemas/canonical_data_layer_vnext_schema_v1.json b/schemas/canonical_table_record_2026.08.12.json similarity index 85% rename from schemas/canonical_data_layer_vnext_schema_v1.json rename to schemas/canonical_table_record_2026.08.12.json index 2cf18b4..ee84ec9 100644 --- a/schemas/canonical_data_layer_vnext_schema_v1.json +++ b/schemas/canonical_table_record_2026.08.12.json @@ -1,10 +1,10 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "mpr-tsr/canonical-data-layer-vnext-v1", - "title": "Canonical Data Layer vNext Record", + "$id": "borderless-table-structuring/canonical-table-record/2026.08.12", + "title": "Canonical Table Record 2026.08.12", "type": "object", "required": [ - "schema_version", + "schema_release", "sample_id", "document_id", "split", @@ -15,7 +15,7 @@ "historical_supervision" ], "properties": { - "schema_version": {"const": "mpr-tsr/canonical-data-layer-vnext-v1"}, + "schema_release": {"const": "2026.08.12"}, "sample_id": {"type": "string", "minLength": 1}, "document_id": {"type": "string", "minLength": 1}, "split": {"type": "string", "minLength": 1}, diff --git a/scripts/compile_canonical_data_layer_vnext.py b/scripts/compile_canonical_data_layer_vnext.py deleted file mode 100644 index 5075980..0000000 --- a/scripts/compile_canonical_data_layer_vnext.py +++ /dev/null @@ -1,427 +0,0 @@ -#!/usr/bin/env python3 -"""Compile the non-overwriting Canonical Data Layer vNext target layer. - -The compiler streams Formal20k v1, preserves every source record by hash and -line reference, and emits direct Canonical Table state plus order-invariant -route targets. It never rewrites the source manifest. -""" - -from __future__ import annotations - -import argparse -from collections import Counter, defaultdict -import hashlib -import json -import os -from pathlib import Path -import sys -import types -from typing import Any, Iterable, Mapping - - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "src")) -PACKAGE = types.ModuleType("mpr_tsr_splitmerge_v2") -PACKAGE.__path__ = [str(ROOT / "src/mpr_tsr_splitmerge_v2")] -sys.modules.setdefault("mpr_tsr_splitmerge_v2", PACKAGE) - -from mpr_tsr_splitmerge_v2.canonical import ( # noqa: E402 - canonical_cells, - normalize_text, - table_shape, - topology_key, - validate_cells, -) - - -SCHEMA = "mpr-tsr/canonical-data-layer-vnext-v1" -EXPECTED_MANIFEST_SHA256 = ( - "716ad18e1c26a58ced016dcf8424049a4081c92e39215ad7b56268b3fd43aa99" -) -EXPECTED_RECORDS = 20_000 -EXPECTED_QUARANTINE = 8 - - -def sha256(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def stable_json(value: Any) -> str: - return json.dumps( - value, ensure_ascii=False, sort_keys=True, separators=(",", ":") - ) - - -def stable_hash(value: Any) -> str: - return hashlib.sha256(stable_json(value).encode("utf-8")).hexdigest() - - -def jsonl_rows(path: Path) -> Iterable[tuple[int, dict[str, Any], str]]: - with path.open(encoding="utf-8") as handle: - for line_number, line in enumerate(handle, 1): - if not line.strip(): - continue - value = json.loads(line) - if not isinstance(value, dict): - raise ValueError(f"{path}:{line_number}: record is not an object") - yield line_number, value, hashlib.sha256(line.encode("utf-8")).hexdigest() - - -def geometry(value: Mapping[str, Any]) -> dict[str, Any]: - bbox = value.get("bbox") - if isinstance(bbox, list) and len(bbox) >= 4: - return { - "status": "present", - "bbox": [round(float(item), 6) for item in bbox[:4]], - } - return {"status": "missing", "bbox": None} - - -def topology_record(value: Mapping[str, Any]) -> dict[str, int]: - row, col, rowspan, colspan = topology_key(dict(value)) - return { - "row_start": row, - "row_end": row + rowspan, - "col_start": col, - "col_end": col + colspan, - } - - -def topology_id(value: Mapping[str, Any]) -> str: - item = topology_record(value) - return ( - f"r{item['row_start']}:{item['row_end']}" - f"c{item['col_start']}:{item['col_end']}" - ) - - -def merge_path_ambiguity(record: Mapping[str, Any]) -> dict[str, int]: - residual = record.get("residual_edit_labels") or {} - per_gold: dict[int, set[int]] = defaultdict(set) - for edge in residual.get("merge_edges", []): - gold_index = int(edge["gold_index"]) - per_gold[gold_index].add(int(edge["source_raw_index"])) - per_gold[gold_index].add(int(edge["target_raw_index"])) - components = sum(len(nodes) >= 3 for nodes in per_gold.values()) - raw_cells = sum(len(nodes) for nodes in per_gold.values() if len(nodes) >= 3) - return {"components": components, "raw_cells": raw_cells} - - -def ownership(record: Mapping[str, Any], gold_count: int) -> list[list[int]]: - """Return Canonical Gold cell ownership from audited text-edit labels. - - The historical ``ocr_gold_pointer`` indexes primitive-grid owners, not - Canonical Gold cells, and therefore must not be interpreted as a Gold - cell index. The text-edit labels expose the replayed Gold index and the - exact OCR token indexes for each Canonical cell. - """ - owners: list[list[int]] = [[] for _ in range(gold_count)] - tokens = list(record.get("ocr_tokens") or []) - valid_token_indexes = { - int(token.get("token_index", position)) - for position, token in enumerate(tokens) - } - assigned: set[int] = set() - text_cells = list((record.get("text_edit_labels") or {}).get("cells") or []) - if len(text_cells) != gold_count: - raise ValueError( - f"text-label cell count {len(text_cells)} != Gold count {gold_count}" - ) - for item in text_cells: - gold_index = int(item.get("gold_index", -1)) - if gold_index < 0 or gold_index >= gold_count: - raise ValueError(f"text-label Gold index {gold_index} is outside range") - for value in item.get("ocr_token_indexes", []): - token_index = int(value) - if token_index not in valid_token_indexes: - raise ValueError(f"OCR token index {token_index} does not exist") - if token_index in assigned: - raise ValueError(f"OCR token index {token_index} has multiple owners") - assigned.add(token_index) - owners[gold_index].append(token_index) - return [sorted(values) for values in owners] - - -def canonical_state( - record: Mapping[str, Any], gold: list[dict[str, Any]] -) -> dict[str, Any]: - owners = ownership(record, len(gold)) - cells: list[dict[str, Any]] = [] - semantic_cells: list[dict[str, Any]] = [] - for gold_index, cell in enumerate(gold): - topo = topology_record(cell) - original_text = str(cell.get("text", "") or "").strip() - normalized = normalize_text(original_text) - tag = "th" if str(cell.get("tag", "td")).lower() == "th" else "td" - semantic = { - "cell_id": topology_id(cell), - **topo, - "normalized_text": normalized, - "tag": tag, - } - semantic_cells.append(semantic) - cells.append( - { - **semantic, - "gold_index": gold_index, - "text": original_text, - "ocr_token_indexes": owners[gold_index], - "geometry": geometry(cell), - } - ) - rows, cols = table_shape(gold) - full = {"rows": rows, "cols": cols, "cells": cells} - return { - **full, - "semantic_state_sha256": stable_hash( - {"rows": rows, "cols": cols, "cells": semantic_cells} - ), - "full_state_sha256": stable_hash(full), - } - - -def explicit_target( - raw: list[dict[str, Any]], gold: list[dict[str, Any]] -) -> dict[str, Any]: - raw_by_id = {topology_id(cell): topology_record(cell) for cell in raw} - gold_by_id = {topology_id(cell): topology_record(cell) for cell in gold} - raw_ids = set(raw_by_id) - gold_ids = set(gold_by_id) - return { - "schema_version": "mpr-tsr/explicit-topology-difference-set-v1", - "target_type": "order_invariant_partition_difference", - "default_action": "KEEP", - "text_policy": "PRESERVE_RAW_BY_DEFAULT", - "preserve": [raw_by_id[key] for key in sorted(raw_ids & gold_ids)], - "remove": [raw_by_id[key] for key in sorted(raw_ids - gold_ids)], - "add": [gold_by_id[key] for key in sorted(gold_ids - raw_ids)], - } - - -def historical_hashes(record: Mapping[str, Any]) -> dict[str, str]: - keys = ( - "structure_labels", - "residual_edit_labels", - "text_candidates", - "text_edit_labels", - "raw_to_gold_diff", - ) - return {key: stable_hash(record.get(key)) for key in keys} - - -def load_quarantine(path: Path) -> set[str]: - value = json.loads(path.read_text(encoding="utf-8")) - if value.get("status") != "complete": - raise ValueError("grid-hole classification is not complete") - sample_ids = {str(row["sample_id"]) for row in value.get("records_detail", [])} - if len(sample_ids) != EXPECTED_QUARANTINE: - raise ValueError( - f"expected {EXPECTED_QUARANTINE} grid-hole IDs, found {len(sample_ids)}" - ) - return sample_ids - - -def atomic_writer(path: Path): - temporary = path.with_name(f".{path.name}.tmp-{os.getpid()}") - return temporary, temporary.open("w", encoding="utf-8") - - -def write_line(handle, value: Mapping[str, Any]) -> None: - handle.write(stable_json(value) + "\n") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--manifest", type=Path, required=True) - parser.add_argument("--grid-hole-classification", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--contract-sha256", required=True) - args = parser.parse_args() - - manifest = args.manifest.resolve() - output = args.output_dir.resolve() - working = output.with_name(f".{output.name}.tmp-{os.getpid()}") - classification = args.grid_hole_classification.resolve() - if output.exists(): - raise SystemExit(f"non-overwrite refusal: output directory exists: {output}") - if working.exists(): - raise SystemExit(f"temporary output directory exists: {working}") - if sha256(manifest) != EXPECTED_MANIFEST_SHA256: - raise SystemExit("frozen Formal20k manifest SHA256 mismatch") - quarantine_ids = load_quarantine(classification) - working.mkdir(parents=True) - - paths = { - "targets": working / "canonical_targets.jsonl", - "replay": working / "replay_map.jsonl", - "included": working / "included_manifest.jsonl", - "quarantined": working / "quarantined_manifest.jsonl", - } - writers = {name: atomic_writer(path) for name, path in paths.items()} - handles = {name: pair[1] for name, pair in writers.items()} - counts: Counter[str] = Counter() - sources: Counter[str] = Counter() - geometry_status: Counter[str] = Counter() - ambiguity_by_source: Counter[str] = Counter() - failures: list[dict[str, Any]] = [] - - try: - for line_number, record, record_sha in jsonl_rows(manifest): - counts["records"] += 1 - sample_id = str(record.get("sample_id") or "") - if not sample_id: - raise ValueError(f"line {line_number}: missing sample_id") - source = str((record.get("metadata") or {}).get("source_dataset") or "unknown") - sources[source] += 1 - raw = canonical_cells( - (record.get("raw_mineru_prior") or {}).get("cells", []), - include_geometry=False, - ) - gold = canonical_cells((record.get("gold") or {}).get("canonical_cells", [])) - raw_errors = validate_cells(raw, require_complete=True) - gold_errors = validate_cells(gold, require_complete=True) - is_quarantined = sample_id in quarantine_ids - if gold_errors: - raise ValueError(f"{sample_id}: invalid Canonical Gold: {gold_errors}") - if raw_errors and not is_quarantined: - raise ValueError(f"{sample_id}: unexpected invalid Raw grid: {raw_errors}") - if is_quarantined and not raw_errors: - raise ValueError(f"{sample_id}: frozen grid-hole record is now Raw-valid") - - ambiguity = merge_path_ambiguity(record) - if ambiguity["components"]: - counts["records_with_historical_action_path_nonuniqueness"] += 1 - counts["historical_nonunique_components"] += ambiguity["components"] - counts["historical_nonunique_raw_cells"] += ambiguity["raw_cells"] - ambiguity_by_source[source] += 1 - - state = canonical_state(record, gold) - geometry_status.update( - cell["geometry"]["status"] for cell in state["cells"] - ) - status = "quarantined" if is_quarantined else "included" - source_record = { - "manifest_path": str(manifest), - "manifest_sha256": EXPECTED_MANIFEST_SHA256, - "line_number": line_number, - "record_sha256": record_sha, - "image_sha256": str((record.get("image") or {}).get("input_image_sha256") or ""), - } - active_target = not is_quarantined - target = { - "schema_version": SCHEMA, - "sample_id": sample_id, - "document_id": str(record.get("document_id") or sample_id), - "split": str(record.get("split") or "unknown"), - "source_dataset": source, - "record_status": status, - "source_record": source_record, - "canonical_table": state, - "explicit_topology_target": explicit_target(raw, gold) if active_target else None, - "lora_canonical_candidate_target": ( - { - "schema_version": "mpr-tsr/lora-canonical-candidate-target-v1", - "scope": "TABLE_ONLY", - "canonical_state_sha256": state["full_state_sha256"], - "geometry_policy": "PARALLEL_OR_SHARED_SIDECAR", - "text_policy": "OCR_COPY_PREFERRED", - } - if active_target - else None - ), - "historical_supervision": { - "primary_truth": False, - "historical_action_path_nonunique": bool(ambiguity["components"]), - "ambiguity": ambiguity, - "artifact_sha256": historical_hashes(record), - }, - "quarantine": ( - { - "policy": "PRESERVE_AND_QUARANTINE", - "reason": "RAW_GRID_HOLE", - "raw_validation_errors": raw_errors, - "requires_separate_repair_evidence": True, - } - if is_quarantined - else None - ), - } - write_line(handles["targets"], target) - replay = { - "sample_id": sample_id, - "source_record_sha256": record_sha, - "historical_artifact_sha256": historical_hashes(record), - "canonical_semantic_state_sha256": state["semantic_state_sha256"], - "canonical_full_state_sha256": state["full_state_sha256"], - "historical_action_path_nonunique": bool(ambiguity["components"]), - } - write_line(handles["replay"], replay) - manifest_row = { - "sample_id": sample_id, - "document_id": str(record.get("document_id") or sample_id), - "source_dataset": source, - "record_status": status, - "canonical_full_state_sha256": state["full_state_sha256"], - } - write_line(handles["quarantined" if is_quarantined else "included"], manifest_row) - counts[status] += 1 - counts["records_processed"] += 1 - except Exception as error: - failures.append({"type": type(error).__name__, "message": str(error)}) - raise - finally: - for handle in handles.values(): - handle.close() - - if counts["records"] != EXPECTED_RECORDS: - raise SystemExit(f"record count mismatch: {counts['records']}") - if counts["quarantined"] != EXPECTED_QUARANTINE: - raise SystemExit(f"quarantine count mismatch: {counts['quarantined']}") - if counts["included"] != EXPECTED_RECORDS - EXPECTED_QUARANTINE: - raise SystemExit(f"included count mismatch: {counts['included']}") - - for name, (temporary, _) in writers.items(): - temporary.replace(paths[name]) - - report = { - "schema_version": "mpr-tsr/canonical-data-layer-vnext-compile-report-v1", - "status": "PASS", - "contract_sha256": args.contract_sha256, - "source_manifest": str(manifest), - "source_manifest_sha256": EXPECTED_MANIFEST_SHA256, - "source_manifest_modified": False, - "terminal_customer50_or_omnidoc_inputs_read": False, - "primary_truth": "DIRECT_CANONICAL_TABLE_STATE", - "grid_hole_policy": "PRESERVE_AND_QUARANTINE", - "counts": dict(sorted(counts.items())), - "sources": dict(sorted(sources.items())), - "geometry_status": dict(sorted(geometry_status.items())), - "historical_action_path_nonuniqueness_by_source": dict( - sorted(ambiguity_by_source.items()) - ), - "failures": failures, - "outputs": { - name: {"path": str(output / path.name), "sha256": sha256(path)} - for name, path in paths.items() - }, - } - report_path = working / "compile_report.json" - report_path.write_text( - json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n", - encoding="utf-8", - ) - sum_paths = [*paths.values(), report_path] - (working / "SHA256SUMS").write_text( - "".join(f"{sha256(path)} {path.name}\n" for path in sum_paths), - encoding="utf-8", - ) - working.replace(output) - print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/src/borderless_table_structuring/__init__.py b/src/borderless_table_structuring/__init__.py new file mode 100644 index 0000000..3e9c8b3 --- /dev/null +++ b/src/borderless_table_structuring/__init__.py @@ -0,0 +1,5 @@ +"""Research primitives for borderless table structure recognition.""" + +__release__ = "2026.08.12" + +__all__ = ["__release__"] diff --git a/src/mpr_tsr_splitmerge_v2/candidate_integrity.py b/src/borderless_table_structuring/candidate_integrity.py similarity index 97% rename from src/mpr_tsr_splitmerge_v2/candidate_integrity.py rename to src/borderless_table_structuring/candidate_integrity.py index 86b9e27..fb404be 100644 --- a/src/mpr_tsr_splitmerge_v2/candidate_integrity.py +++ b/src/borderless_table_structuring/candidate_integrity.py @@ -69,12 +69,12 @@ def _index_sidecars( return indexed -def audit_local_candidate_integrity( +def audit_candidate_integrity( manifest: Path, sidecar_dir: Path, candidate_audit: dict[str, Any], *, - expected_tables: int = 20000, + expected_tables: int, ) -> dict[str, Any]: issues: Counter[str] = Counter() totals: Counter[str] = Counter() @@ -117,7 +117,7 @@ def audit_local_candidate_integrity( "gold_text_visible_to_recognizer": False, "gold_geometry_visible_to_recognizer": False, "candidate_text_frozen_before_gold_matching": True, - "terminal_benchmarks_visible": False, + "restricted_evaluation_visible": False, "crop_geometry_source": "frozen_paddleocr_token_bbox_only", "logical_projection_bbox_forbidden": True, "candidate_proposal_policy": expected_policy, @@ -245,7 +245,7 @@ def audit_local_candidate_integrity( ), } return { - "schema_version": "mpr-tsr/formal20k-local-candidate-integrity-v1", + "schema_release": "candidate-integrity-2026.08.12", "status": "PASS" if all(checks.values()) else "FAIL", "checks": checks, "manifest": str(manifest.resolve()), @@ -258,6 +258,5 @@ def audit_local_candidate_integrity( for name, count in sorted(view_count_distribution.items()) }, "issues": dict(sorted(issues.items())), - "customer50_visible": False, - "omnidocbench_visible": False, + "restricted_evaluation_visible": False, } diff --git a/src/mpr_tsr_splitmerge_v2/candidate_interfaces.py b/src/borderless_table_structuring/candidate_interfaces.py similarity index 95% rename from src/mpr_tsr_splitmerge_v2/candidate_interfaces.py rename to src/borderless_table_structuring/candidate_interfaces.py index 41b6bda..f877025 100644 --- a/src/mpr_tsr_splitmerge_v2/candidate_interfaces.py +++ b/src/borderless_table_structuring/candidate_interfaces.py @@ -12,14 +12,14 @@ EXPLICIT_POLICY = SafetyPolicy( - policy_id="stage3-explicit-topology-only-v1", + policy_id="explicit-topology-only-2026.08.12", minimum_expected_gain=0.0, threshold_source="NONTERMINAL_PREREGISTERED", text_policy="OCR_GROUNDED", ) LORA_POLICY = SafetyPolicy( - policy_id="stage3-lora-table-only-v1", + policy_id="lora-table-only-2026.08.12", minimum_expected_gain=0.0, threshold_source="NONTERMINAL_PREREGISTERED", text_policy="OCR_GROUNDED", @@ -87,10 +87,10 @@ def _route_provenance( return { "sample_id": str(raw.get("sample_id", "")), "producer": producer, - "producer_version": "v1", + "producer_release": "2026.08.12", "purpose": purpose, "input_image_sha256": str(raw.get("input_image_sha256", "")), - "terminal_benchmarks_visible": False, + "restricted_evaluation_visible": False, "raw_state_sha256": stable_sha256(raw_record), } @@ -163,7 +163,7 @@ def build_explicit_topology_candidate( candidate_rows = int(rows if rows is not None else raw_table["rows"]) candidate_cols = int(cols if cols is not None else raw_table["cols"]) return { - "schema_version": "mpr-tsr/explicit-topology-candidate-v1", + "schema_release": "explicit-topology-candidate-2026.08.12", "canonical_table": { "rows": candidate_rows, "cols": candidate_cols, @@ -177,7 +177,7 @@ def build_explicit_topology_candidate( "full_page_rewrite": False, }, "replay": { - "schema_version": "mpr-tsr/explicit-reversible-replay-v1", + "schema_release": "explicit-reversible-replay-2026.08.12", "raw_state_sha256": stable_sha256(raw_record), "partitions": replay_partitions, }, @@ -211,7 +211,7 @@ def build_lora_table_candidate( if not isinstance(table, dict) or not isinstance(table.get("cells"), list): raise ValueError("LoRA output must contain one complete Canonical Table") return { - "schema_version": "mpr-tsr/lora-canonical-table-candidate-v1", + "schema_release": "lora-canonical-table-candidate-2026.08.12", "canonical_table": table, "candidate_interface": { "route": "LORA_TABLE_MODEL", diff --git a/src/mpr_tsr_splitmerge_v2/canonical.py b/src/borderless_table_structuring/canonical.py similarity index 100% rename from src/mpr_tsr_splitmerge_v2/canonical.py rename to src/borderless_table_structuring/canonical.py diff --git a/src/borderless_table_structuring/explicit.py b/src/borderless_table_structuring/explicit.py new file mode 100644 index 0000000..9212a46 --- /dev/null +++ b/src/borderless_table_structuring/explicit.py @@ -0,0 +1,20 @@ +"""Public research interface for the Explicit Layout Transformer route. + +The trainable architecture is deliberately decoupled from the canonical +candidate contract. Architecture variants should emit hypotheses through this +module so representation, replay, and evaluation remain comparable. +""" + +from .candidate_interfaces import ( + EXPLICIT_POLICY, + build_explicit_topology_candidate, + replay_explicit_to_raw, + select_explicit_candidate, +) + +__all__ = [ + "EXPLICIT_POLICY", + "build_explicit_topology_candidate", + "replay_explicit_to_raw", + "select_explicit_candidate", +] diff --git a/src/mpr_tsr_splitmerge_v2/labels.py b/src/borderless_table_structuring/labels.py similarity index 98% rename from src/mpr_tsr_splitmerge_v2/labels.py rename to src/borderless_table_structuring/labels.py index 6da80d3..e93d4ef 100644 --- a/src/mpr_tsr_splitmerge_v2/labels.py +++ b/src/borderless_table_structuring/labels.py @@ -16,8 +16,8 @@ CONTROL_POINTS = 4 -HEADER_POLICY_VERSION = "gold-functional-th-td/v1" -RESIDUAL_EDIT_POLICY_VERSION = "logical-topology-minimal-edit/v1" +HEADER_POLICY_RELEASE = "gold-functional-th-td/2026.08.12" +RESIDUAL_EDIT_POLICY_RELEASE = "logical-topology-minimal-edit/2026.08.12" def _logical_box( @@ -242,7 +242,7 @@ def compile_residual_edit_labels( text_edit = any(item["text_update"] for item in per_raw) header_edit = any(item["header_update"] for item in per_raw) return { - "policy_version": RESIDUAL_EDIT_POLICY_VERSION, + "policy_release": RESIDUAL_EDIT_POLICY_RELEASE, "alignment_basis": "normalized_logical_topology_only", "physical_bbox_used": False, "confidence_used": False, @@ -271,7 +271,7 @@ def compile_residual_edit_labels( "cell_payload": structure_labels["cell_payload"], }, "header_policy": { - "version": HEADER_POLICY_VERSION, + "release": HEADER_POLICY_RELEASE, "target_source": "public_gold_only", "classes": ["td", "th"], "raw_mineru_tag_is_observation_not_target": True, @@ -499,7 +499,7 @@ def compile_structure_labels( "merge_down": merge_down, "header": header, "header_policy": { - "version": HEADER_POLICY_VERSION, + "release": HEADER_POLICY_RELEASE, "target_source": "public_gold_only", "classes": ["td", "th"], }, diff --git a/src/mpr_tsr_splitmerge_v2/safety_layer.py b/src/borderless_table_structuring/safety_layer.py similarity index 96% rename from src/mpr_tsr_splitmerge_v2/safety_layer.py rename to src/borderless_table_structuring/safety_layer.py index f932880..d4fdffa 100644 --- a/src/mpr_tsr_splitmerge_v2/safety_layer.py +++ b/src/borderless_table_structuring/safety_layer.py @@ -129,15 +129,15 @@ def _provenance_issues(record: dict[str, Any]) -> list[str]: if not isinstance(provenance, dict): return ["PROVENANCE_MISSING"] issues: list[str] = [] - for field in ("sample_id", "producer", "producer_version", "purpose", "input_image_sha256"): + for field in ("sample_id", "producer", "producer_release", "purpose", "input_image_sha256"): if not str(provenance.get(field, "")).strip(): issues.append(f"PROVENANCE_{field.upper()}_MISSING") - if provenance.get("terminal_benchmarks_visible") is not False: - issues.append("PROVENANCE_TERMINAL_VISIBILITY_NOT_FALSE") - terminal_words = ("customer50", "omnidocbench") + if provenance.get("restricted_evaluation_visible") is not False: + issues.append("PROVENANCE_RESTRICTED_EVALUATION_VISIBILITY_NOT_FALSE") + terminal_words = ("private-evaluation", "restricted-evaluation") serialized = json.dumps(provenance, ensure_ascii=False).lower() if any(word in serialized for word in terminal_words): - issues.append("PROVENANCE_TERMINAL_REFERENCE_FORBIDDEN") + issues.append("PROVENANCE_RESTRICTED_EVALUATION_REFERENCE_FORBIDDEN") return issues @@ -248,7 +248,7 @@ def validate_candidate( ) unique_issues = sorted(set(issues)) return { - "schema_version": "mpr-tsr/shared-safety-validation-v1", + "schema_release": "shared-candidate-validation-2026.08.12", "status": "PASS" if not unique_issues else "FAIL", "issues": unique_issues, "cell_count": len(cells), @@ -356,7 +356,7 @@ def assemble_table_only( if not frozen_non_table: raise RuntimeError("Non-table page state changed during table assembly") return { - "schema_version": "mpr-tsr/table-only-assembly-v1", + "schema_release": "table-only-assembly-2026.08.12", "status": "PASS", "decision": selection.get("decision"), "output_page": output_page, @@ -376,7 +376,7 @@ def _selection( ) -> dict[str, Any]: output_hash = stable_sha256(output) return { - "schema_version": "mpr-tsr/shared-safety-selection-v1", + "schema_release": "shared-candidate-selection-2026.08.12", "decision": decision, "reason_codes": sorted(set(reasons)), "output": output, diff --git a/src/mpr_tsr_splitmerge_v2/__init__.py b/src/mpr_tsr_splitmerge_v2/__init__.py deleted file mode 100644 index 729570e..0000000 --- a/src/mpr_tsr_splitmerge_v2/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -"""Auditable data-engineering primitives for borderless table structuring.""" - -__all__: list[str] = [] diff --git a/tests/test_candidate_integrity.py b/tests/test_candidate_integrity.py index cab7709..44e9848 100644 --- a/tests/test_candidate_integrity.py +++ b/tests/test_candidate_integrity.py @@ -3,8 +3,8 @@ import json from pathlib import Path -from mpr_tsr_splitmerge_v2.candidate_integrity import ( - audit_local_candidate_integrity, +from borderless_table_structuring.candidate_integrity import ( + audit_candidate_integrity, ) @@ -58,10 +58,10 @@ def _fixture(tmp_path: Path) -> tuple[Path, Path, dict[str, object]]: "gold_text_visible_to_recognizer": False, "gold_geometry_visible_to_recognizer": False, "candidate_text_frozen_before_gold_matching": True, - "terminal_benchmarks_visible": False, + "restricted_evaluation_visible": False, "crop_geometry_source": "frozen_paddleocr_token_bbox_only", "logical_projection_bbox_forbidden": True, - "candidate_proposal_policy": "policy-v1", + "candidate_proposal_policy": "proposal-policy-2026.08.12", "candidate_proposal_mode": "low_confidence_ocr_token_lines", "proposal_gate_threshold": 0.05, "token_confidence_threshold": 0.95, @@ -79,7 +79,7 @@ def _fixture(tmp_path: Path) -> tuple[Path, Path, dict[str, object]]: "status": "PASS", "token_confidence_threshold": 0.95, "maximum_candidates_per_table": 1180, - "candidate_proposal_policy": "policy-v1", + "candidate_proposal_policy": "proposal-policy-2026.08.12", "candidate_proposal_mode": "low_confidence_ocr_token_lines", "proposal_gate_threshold": 0.05, "frozen_config_sha256": "config-sha", @@ -93,7 +93,7 @@ def test_candidate_integrity_accepts_complete_gold_free_sidecar( tmp_path: Path, ) -> None: manifest, sidecar_dir, audit = _fixture(tmp_path) - report = audit_local_candidate_integrity( + report = audit_candidate_integrity( manifest, sidecar_dir, audit, expected_tables=1 ) assert report["status"] == "PASS" @@ -110,7 +110,7 @@ def test_candidate_integrity_rejects_gold_field_and_bbox_misbinding( value["gold_text"] = "leak" value["token_candidates"][0]["absolute_bbox"] = [2, 2, 30, 12] _write_json(path, value) - report = audit_local_candidate_integrity( + report = audit_candidate_integrity( manifest, sidecar_dir, audit, expected_tables=1 ) assert report["status"] == "FAIL" @@ -127,7 +127,7 @@ def test_candidate_integrity_rejects_missing_eligible_token( value["token_candidates"] = [] value["proposal_count"] = 0 _write_json(path, value) - report = audit_local_candidate_integrity( + report = audit_candidate_integrity( manifest, sidecar_dir, audit, expected_tables=1 ) assert report["status"] == "FAIL" diff --git a/tests/test_candidate_interfaces.py b/tests/test_candidate_interfaces.py index 606cbbd..9e18451 100644 --- a/tests/test_candidate_interfaces.py +++ b/tests/test_candidate_interfaces.py @@ -4,18 +4,18 @@ import pytest -from mpr_tsr_splitmerge_v2.candidate_interfaces import ( +from borderless_table_structuring.candidate_interfaces import ( build_explicit_topology_candidate, build_lora_table_candidate, replay_explicit_to_raw, select_explicit_candidate, select_lora_candidate, ) -from mpr_tsr_splitmerge_v2.safety_layer import ExpectedGainEvidence, stable_sha256 +from borderless_table_structuring.safety_layer import ExpectedGainEvidence, stable_sha256 OCR = [{"text": "A"}, {"text": "B"}] -GAIN = ExpectedGainEvidence(0.1, "stage3-interface-fixture-v1", "NONTERMINAL_DEVELOPMENT") +GAIN = ExpectedGainEvidence(0.1, "interface-fixture-2026.08.12", "NONTERMINAL_DEVELOPMENT") def _raw_record(): @@ -51,10 +51,10 @@ def _raw_record(): "provenance": { "sample_id": "isolated-dev-fixture-001", "producer": "raw-mineru-fixture", - "producer_version": "v1", - "purpose": "nonterminal_stage3_fixture", + "producer_release": "2026.08.12", + "purpose": "synthetic_interface_fixture", "input_image_sha256": "0" * 64, - "terminal_benchmarks_visible": False, + "restricted_evaluation_visible": False, }, } diff --git a/tests/test_canonical_data_layer_vnext.py b/tests/test_canonical_data_layer_vnext.py deleted file mode 100644 index 3a4a716..0000000 --- a/tests/test_canonical_data_layer_vnext.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -import importlib.util -from pathlib import Path - - -ROOT = Path(__file__).resolve().parents[1] -SPEC = importlib.util.spec_from_file_location( - "compile_canonical_data_layer_vnext", - ROOT / "scripts" / "compile_canonical_data_layer_vnext.py", -) -assert SPEC and SPEC.loader -MODULE = importlib.util.module_from_spec(SPEC) -SPEC.loader.exec_module(MODULE) - - -def test_explicit_target_is_order_invariant_and_defaults_to_keep() -> None: - raw = [ - {"row": 0, "col": 1, "rowspan": 1, "colspan": 1}, - {"row": 0, "col": 0, "rowspan": 1, "colspan": 1}, - ] - gold = [ - {"row": 0, "col": 0, "rowspan": 1, "colspan": 2}, - ] - first = MODULE.explicit_target(raw, gold) - second = MODULE.explicit_target(list(reversed(raw)), list(reversed(gold))) - assert first == second - assert first["default_action"] == "KEEP" - assert len(first["remove"]) == 2 - assert len(first["add"]) == 1 - - -def test_ownership_rejects_out_of_range_gold_index() -> None: - record = { - "ocr_tokens": [{"token_index": 0}], - "text_edit_labels": { - "cells": [{"gold_index": 3, "ocr_token_indexes": [0]}] - }, - } - try: - MODULE.ownership(record, 1) - except ValueError as error: - assert "outside range" in str(error) - else: - raise AssertionError("out-of-range pointer was accepted") - - -def test_canonical_state_hash_is_independent_of_input_cell_order() -> None: - record = { - "ocr_tokens": [], - "text_edit_labels": { - "cells": [ - {"gold_index": 0, "ocr_token_indexes": []}, - {"gold_index": 1, "ocr_token_indexes": []}, - ] - }, - } - cells = [ - {"row": 0, "col": 0, "rowspan": 1, "colspan": 1, "text": "A", "tag": "th"}, - {"row": 0, "col": 1, "rowspan": 1, "colspan": 1, "text": "B", "tag": "td"}, - ] - first = MODULE.canonical_state(record, MODULE.canonical_cells(cells)) - second = MODULE.canonical_state(record, MODULE.canonical_cells(reversed(cells))) - assert first["semantic_state_sha256"] == second["semantic_state_sha256"] - assert first["full_state_sha256"] == second["full_state_sha256"] diff --git a/tests/test_explicit_route.py b/tests/test_explicit_route.py new file mode 100644 index 0000000..b8fd8c7 --- /dev/null +++ b/tests/test_explicit_route.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from copy import deepcopy + +from borderless_table_structuring.explicit import ( + build_explicit_topology_candidate, + replay_explicit_to_raw, +) + + +def _raw_record() -> dict[str, object]: + return { + "canonical_table": { + "rows": 1, + "cols": 2, + "cells": [ + { + "cell_id": "left", + "row_start": 0, + "row_end": 1, + "col_start": 0, + "col_end": 1, + "text": "A", + "tag": "td", + "ocr_token_indexes": [0], + "geometry": {"status": "present", "bbox": [0, 0, 10, 10]}, + }, + { + "cell_id": "right", + "row_start": 0, + "row_end": 1, + "col_start": 1, + "col_end": 2, + "text": "B", + "tag": "td", + "ocr_token_indexes": [1], + "geometry": {"status": "present", "bbox": [10, 0, 20, 10]}, + }, + ], + }, + "provenance": { + "sample_id": "synthetic-explicit-fixture", + "input_image_sha256": "0" * 64, + }, + } + + +def test_public_explicit_interface_preserves_identity() -> None: + raw = _raw_record() + candidate = build_explicit_topology_candidate( + raw, + partitions=None, + ocr_tokens=[{"text": "A"}, {"text": "B"}], + ) + assert candidate == raw + assert candidate is not raw + + +def test_public_explicit_interface_is_reversible() -> None: + raw = _raw_record() + raw_snapshot = deepcopy(raw) + candidate = build_explicit_topology_candidate( + raw, + partitions=[ + { + "cell_id": "merged", + "source_cell_ids": ["left", "right"], + "row_start": 0, + "row_end": 1, + "col_start": 0, + "col_end": 2, + } + ], + ocr_tokens=[{"text": "A"}, {"text": "B"}], + ) + assert candidate["canonical_table"]["cells"][0]["text"] == "AB" + assert replay_explicit_to_raw(raw, candidate) == raw_snapshot diff --git a/tests/test_safety_layer.py b/tests/test_safety_layer.py index 625093b..3339cbe 100644 --- a/tests/test_safety_layer.py +++ b/tests/test_safety_layer.py @@ -2,7 +2,7 @@ from copy import deepcopy -from mpr_tsr_splitmerge_v2.safety_layer import ( +from borderless_table_structuring.safety_layer import ( ExpectedGainEvidence, SafetyPolicy, assemble_table_only, @@ -40,16 +40,16 @@ def _record(cells=None): "provenance": { "sample_id": "dev-fixture-001", "producer": "unit-fixture", - "producer_version": "v1", + "producer_release": "2026.08.12", "purpose": "nonterminal_correctness_fixture", "input_image_sha256": "0" * 64, - "terminal_benchmarks_visible": False, + "restricted_evaluation_visible": False, }, } POLICY = SafetyPolicy( - policy_id="stage2-preregistered-v1", + policy_id="synthetic-preregistered-2026.08.12", minimum_expected_gain=0.0, threshold_source="NONTERMINAL_PREREGISTERED", text_policy="OCR_GROUNDED", @@ -73,7 +73,7 @@ def test_overlap_fails_and_rolls_back_exactly(): raw, candidate, policy=POLICY, - expected_gain=ExpectedGainEvidence(1.0, "dev-protocol-v1", "NONTERMINAL_DEVELOPMENT"), + expected_gain=ExpectedGainEvidence(1.0, "development-protocol-2026.08.12", "NONTERMINAL_DEVELOPMENT"), ocr_tokens=OCR, ) assert result["decision"] == "ROLLBACK" @@ -112,14 +112,14 @@ def test_valid_changed_candidate_needs_positive_nonterminal_gain(): raw, candidate, policy=POLICY, - expected_gain=ExpectedGainEvidence(0.0, "dev-protocol-v1", "NONTERMINAL_DEVELOPMENT"), + expected_gain=ExpectedGainEvidence(0.0, "development-protocol-2026.08.12", "NONTERMINAL_DEVELOPMENT"), ocr_tokens=OCR, ) accepted = select_candidate_or_rollback( raw, candidate, policy=POLICY, - expected_gain=ExpectedGainEvidence(0.1, "dev-protocol-v1", "NONTERMINAL_DEVELOPMENT"), + expected_gain=ExpectedGainEvidence(0.1, "development-protocol-2026.08.12", "NONTERMINAL_DEVELOPMENT"), ocr_tokens=OCR, ) assert missing["decision"] == "ROLLBACK" @@ -132,11 +132,11 @@ def test_missing_geometry_and_bad_provenance_fail(): raw = _record() candidate = deepcopy(raw) candidate["canonical_table"]["cells"][0].pop("geometry") - candidate["provenance"]["terminal_benchmarks_visible"] = True + candidate["provenance"]["restricted_evaluation_visible"] = True validation = validate_candidate(raw, candidate, policy=POLICY, ocr_tokens=OCR) assert validation["status"] == "FAIL" assert "GEOMETRY_MISSING_OR_INVALID" in validation["issues"] - assert "PROVENANCE_TERMINAL_VISIBILITY_NOT_FALSE" in validation["issues"] + assert "PROVENANCE_RESTRICTED_EVALUATION_VISIBILITY_NOT_FALSE" in validation["issues"] def test_table_only_assembler_freezes_every_other_page_block(): @@ -158,7 +158,7 @@ def test_table_only_assembler_freezes_every_other_page_block(): raw, candidate, policy=POLICY, - expected_gain=ExpectedGainEvidence(0.1, "dev-protocol-v1", "NONTERMINAL_DEVELOPMENT"), + expected_gain=ExpectedGainEvidence(0.1, "development-protocol-2026.08.12", "NONTERMINAL_DEVELOPMENT"), ocr_tokens=OCR, ) page = {