Add frozen shared corpus builder

README.md

@@ -26,14 +26,15 @@ OmniDocBench is used as one document-parsing evaluation protocol. The methods
 and infrastructure in this repository are designed around the broader research
 problem of table structure recognition.
 
-## Research snapshot 2026.08.12.4
+## Research snapshot 2026.08.12.5
 
-The `2026.08.12.4` snapshot establishes the shared representation, evaluation,
-and synthetic-corpus specification for two independent modeling tracks. It
+The `2026.08.12.5` snapshot establishes the shared representation, evaluation,
+and frozen shared-corpus builder for two independent modeling tracks. It
 includes Canonical Table normalization, order-invariant topology targets,
-candidate-integrity checks, table-only model interfaces, a frozen 256-record
-generator-smoke design, and data-free regression tests. Model checkpoints and
-dataset payloads are maintained outside this repository.
+candidate-integrity checks, table-only model interfaces, a sealed 256-record
+generator smoke, a preregistered 40,000-record corpus design, scalable overlap
+audits, and data-free regression tests. Model checkpoints and dataset payloads
+are maintained outside this repository.
 
 Project-authored releases follow calendar versioning:
 
@@ -107,6 +108,7 @@ borderless-table-structuring-lab/
 ├── scripts/                   # Calendar-versioned corpus entry points
 ├── src/borderless_table_structuring/
 │   ├── canonical.py           # Canonical table normalization
+│   ├── shared_corpus.py       # Streaming shared-corpus build and audit
 │   ├── explicit.py            # Public Explicit-route interface
 │   ├── candidate_interfaces.py
 │   ├── candidate_integrity.py
@@ -145,7 +147,7 @@ compression, blur, background, and scanning artifacts.
 Dataset roles are assigned by document, template, content, renderer, and seed
 families before rendering. Exact and near-duplicate audits operate on images,
 text, normalized structure, geometry, and provenance. See the
-[synthetic data specification](docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.4.md)
+[shared-corpus specification](docs/corpus/SHARED_CORPUS_SPECIFICATION_2026.08.12.md)
 and [data governance guide](docs/corpus/DATA_GOVERNANCE_2026.08.12.1.md).
 
 ## Installation
@@ -174,6 +176,17 @@ python scripts/generate_data_smoke_2026.08.12.4.py \
   --output /absolute/path/to/data-smoke-2026.08.12.4
 ```
 
+After independently verifying that sealed smoke, build the preregistered
+shared corpus into a different new external path:
+
+```bash
+python scripts/build_shared_corpus_2026.08.12.py \
+  --output /absolute/path/to/shared-corpus-2026.08.12
+```
+
+The builder is CPU-first, streams the 40,000 records, and refuses an existing
+output path. Use `--verify-only` for an independent payload and checksum pass.
+
 ## Reproducible research
 
 Experiments record the code revision, schema release, immutable data revision,

configs/shared_corpus_parameters_2026.08.12.json

@@ -0,0 +1,62 @@
+{
+  "configuration_release": "2026.08.12",
+  "dataset_release": "shared-corpus-2026.08.12",
+  "generator_release": "2026.08.12.5",
+  "schema_release": "synthetic-table-record-2026.08.12.5",
+  "seed_namespace": "borderless-table-structuring-shared-corpus-2026.08.12",
+  "seed_start": 202608120000000,
+  "requested_records": 40000,
+  "allow_failed_seed_replacement": false,
+  "gate_costs": {
+    "false_edit": 4.0,
+    "missed_edit": 1.0,
+    "uncertain_action": "KEEP"
+  },
+  "categories": {
+    "exact_keep": {"count": 10000, "gate_label": "KEEP"},
+    "hard_keep": {"count": 10000, "gate_label": "KEEP"},
+    "single_minimal_edit": {"count": 10000, "gate_label": "EDIT"},
+    "complex_correction": {"count": 10000, "gate_label": "EDIT"}
+  },
+  "roles_per_category": {
+    "train": 7000,
+    "development": 2000,
+    "holdout": 1000
+  },
+  "counterfactual_pairs": {
+    "minimum_pairs": 10000,
+    "same_role_required": true,
+    "shared_fields": [
+      "content_family_id",
+      "template_family_id",
+      "font_family_id",
+      "renderer_family_id",
+      "base_degradation"
+    ]
+  },
+  "structure": {
+    "rows": {"minimum": 3, "maximum": 28},
+    "columns": {"minimum": 2, "maximum": 14},
+    "header_depth": {"minimum": 1, "maximum": 4},
+    "maximum_cells": 240
+  },
+  "corruptions": {
+    "single_minimal_edit_exactly_one": true,
+    "complex_correction_minimum_operations": 2,
+    "complex_correction_maximum_operations": 4,
+    "accepted_prior_must_remain_canonical_legal": true,
+    "invalid_grid_fixtures_are_test_only": true
+  },
+  "determinism": {
+    "semantic_replay_runs": 2,
+    "normalized_pixel_replay_runs": 2,
+    "sha256_algorithm": "sha256"
+  },
+  "execution": {
+    "cpu_first": true,
+    "stream_records": true,
+    "stream_checksums": true,
+    "maximum_records_in_memory": 2,
+    "progress_interval_records": 500
+  }
+}

docs/corpus/SHARED_CORPUS_COVERAGE_2026.08.12.csv

@@ -0,0 +1,30 @@
+category,gate_label,phenomenon,requested_count,train_count,development_count,holdout_count,counterfactual_pair_required,primary_risk
+exact_keep,KEEP,simple_regular,1875,1312,375,188,false,false_edit
+exact_keep,KEEP,hierarchical_header,1875,1313,375,187,false,false_edit
+exact_keep,KEEP,row_or_column_span,1875,1312,375,188,false,false_edit
+exact_keep,KEEP,empty_or_sparse_cells,1250,875,250,125,false,false_edit
+exact_keep,KEEP,dense_or_multiline_text,1250,875,250,125,false,text_loss
+exact_keep,KEEP,mixed_script_or_formula,937,656,187,94,false,text_loss
+exact_keep,KEEP,clean_high_resolution_control,938,657,188,93,false,false_edit
+hard_keep,KEEP,multilevel_irregular_header,1875,1312,375,188,true,false_edit
+hard_keep,KEEP,mixed_two_dimensional_spans,1875,1313,375,187,true,false_edit
+hard_keep,KEEP,weak_missing_or_partial_borders,1875,1312,375,188,true,false_edit
+hard_keep,KEEP,empty_near_empty_or_sparse,1250,875,250,125,true,false_edit
+hard_keep,KEEP,dense_small_long_or_multiline_text,1250,875,250,125,true,text_loss
+hard_keep,KEEP,rotation_blur_compression_or_noise,1250,875,250,125,true,false_edit
+hard_keep,KEEP,color_background_or_alignment_trap,625,438,125,62,true,false_edit
+single_minimal_edit,EDIT,extra_split,1875,1312,375,188,true,cell_inflation
+single_minimal_edit,EDIT,missing_split,1563,1094,313,156,true,under_segmentation
+single_minimal_edit,EDIT,incorrect_merge,1562,1094,312,156,true,content_reassignment
+single_minimal_edit,EDIT,missing_merge,1563,1094,313,156,true,over_segmentation
+single_minimal_edit,EDIT,span_extent_error,1250,875,250,125,true,topology_error
+single_minimal_edit,EDIT,token_ownership_error,937,656,187,94,true,text_loss
+single_minimal_edit,EDIT,geometry_inconsistency,625,438,125,62,true,location_error
+single_minimal_edit,EDIT,row_or_column_assignment_error,625,437,125,63,true,topology_error
+complex_correction,EDIT,multilevel_header_with_spans,1875,1312,375,188,false,topology_error
+complex_correction,EDIT,joint_split_and_merge,1875,1313,375,187,false,cell_inflation
+complex_correction,EDIT,weak_border_with_topology_error,1563,1094,313,156,false,missed_edit
+complex_correction,EDIT,empty_cells_with_spans,1250,875,250,125,false,ownership_error
+complex_correction,EDIT,dense_multiline_with_topology_error,1250,875,250,125,false,text_loss
+complex_correction,EDIT,mixed_script_formula_with_topology_error,937,656,187,94,false,text_loss
+complex_correction,EDIT,rotation_noise_with_topology_error,1250,875,250,125,false,geometry_error

docs/corpus/SHARED_CORPUS_SPECIFICATION_2026.08.12.md

@@ -0,0 +1,82 @@
+# Shared Corpus Specification 2026.08.12
+
+Status: `FROZEN_FOR_NON_OVERWRITING_BUILD`
+
+Dataset identifier: `shared-corpus-2026.08.12`
+
+Generator identifier: `2026.08.12.5`
+
+Schema identifier: `synthetic-table-record-2026.08.12.5`
+
+## Research purpose
+
+This release builds a shared, source-traceable corpus for independent Explicit
+Layout Transformer and LoRA Table Model research. It emphasizes conservative
+KEEP behavior, direct final Canonical Table supervision, and minimal legal
+structural correction without using terminal benchmark content.
+
+## Frozen scale and roles
+
+| Category | Train | Development | Holdout | Total |
+|---|---:|---:|---:|---:|
+| Exact KEEP | 7,000 | 2,000 | 1,000 | 10,000 |
+| Hard KEEP | 7,000 | 2,000 | 1,000 | 10,000 |
+| Single Minimal Edit | 7,000 | 2,000 | 1,000 | 10,000 |
+| Complex Correction | 7,000 | 2,000 | 1,000 | 10,000 |
+| **Total** | **28,000** | **8,000** | **4,000** | **40,000** |
+
+Hard KEEP and Single Minimal Edit form 10,000 same-role counterfactual pairs.
+Each pair shares one rendered observation, one complete Gold state, and all
+declared family identities. The KEEP member uses Gold unchanged. The EDIT
+member uses one legal, non-identity Raw-like prior.
+
+## Supervision views
+
+- Gate: KEEP or EDIT, with false editing weighted four times a missed edit.
+- Explicit: Raw-like prior, Raw text frozen, and an order-invariant structural
+  difference to Gold.
+- LoRA: one complete table-only Canonical Table target.
+
+An arbitrary ordered merge/split program is not a primary target.
+
+## Source and license boundary
+
+All accepted content is project-authored synthetic content rendered with
+rebuild-only runtime assets recorded in the source-license manifest. No
+OmniDocBench or Customer50 page, crop, image, string, coordinate, annotation,
+identifier, embedding, or transformed derivative may enter the build.
+
+## Isolation
+
+Role assignment occurs before rendering. Document, source, template, content,
+renderer, font, and seed families are role-scoped. The build must report exact
+payload/table matches, family overlaps, perceptual near matches with supporting
+text or geometry evidence, and normalized structure/text/geometry signature
+statistics. Any cross-role exact or unresolved near overlap is a hard stop.
+
+## Execution
+
+The generator is CPU-first and streams records, images, checksums, and compact
+audit signatures. It must never retain more than two full records in memory.
+The output path is new and non-overwriting. Failed seeds are not replaced.
+
+## Acceptance
+
+- Requested, generated, and accepted: 40,000.
+- Quarantined and failed: 0.
+- Exact KEEP / Hard KEEP / Single / Complex: 10,000 each.
+- Train / development / holdout: 28,000 / 8,000 / 4,000.
+- Complete same-role counterfactual pairs: 10,000.
+- Schema, Canonical legality, complete grid, token ownership, geometry,
+  deterministic semantic replay, and deterministic normalized-pixel replay:
+  40,000/40,000.
+- KEEP identity: 20,000/20,000.
+- EDIT legal non-identity replay: 20,000/20,000.
+- Cross-role overlap audit: PASS.
+- Terminal inputs used: false.
+- Complete SHA256 manifest and independent verification: PASS.
+
+## Boundary
+
+A sealed corpus authorizes bounded, preregistered candidate pilots only. It
+does not authorize full model training.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "borderless-table-structuring-lab"
-version = "2026.8.12.4"
+version = "2026.8.12.5"
 description = "Research on canonical representations, explicit topology modeling, and generative adaptation for borderless table structure recognition."
 readme = "README.md"
 requires-python = ">=3.10"

schemas/synthetic_table_record_2026.08.12.5.json

@@ -0,0 +1,91 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "borderless-table-structuring/synthetic-table-record/2026.08.12.5",
+  "title": "Synthetic Table Record 2026.08.12.5",
+  "type": "object",
+  "required": [
+    "schema_release",
+    "dataset_release",
+    "sample_id",
+    "role",
+    "gate_label",
+    "category",
+    "identity",
+    "provenance",
+    "rendering",
+    "payloads",
+    "gold",
+    "prior",
+    "views",
+    "audits"
+  ],
+  "properties": {
+    "schema_release": {"const": "synthetic-table-record-2026.08.12.5"},
+    "dataset_release": {"const": "shared-corpus-2026.08.12"},
+    "sample_id": {"type": "string", "pattern": "^shared-corpus-2026\\.08\\.12-[0-9]{6}$"},
+    "role": {"enum": ["train", "development", "holdout"]},
+    "gate_label": {"enum": ["KEEP", "EDIT", "QUARANTINE"]},
+    "category": {"enum": ["exact_keep", "hard_keep", "single_minimal_edit", "complex_correction"]},
+    "identity": {
+      "type": "object",
+      "required": ["generation_seed", "document_cluster_id", "source_family_id", "template_family_id", "content_family_id", "renderer_family_id", "font_family_id", "counterfactual_pair_id"],
+      "properties": {
+        "generation_seed": {"type": "integer", "minimum": 0},
+        "document_cluster_id": {"type": "string", "minLength": 1},
+        "source_family_id": {"type": "string", "minLength": 1},
+        "template_family_id": {"type": "string", "minLength": 1},
+        "content_family_id": {"type": "string", "minLength": 1},
+        "renderer_family_id": {"type": "string", "minLength": 1},
+        "font_family_id": {"type": "string", "minLength": 1},
+        "counterfactual_pair_id": {"type": ["string", "null"]}
+      },
+      "additionalProperties": false
+    },
+    "provenance": {
+      "type": "object",
+      "required": ["generator_release", "source_id", "license_decision", "license_manifest_sha256", "terminal_inputs_used"],
+      "properties": {
+        "generator_release": {"const": "2026.08.12.5"},
+        "source_id": {"type": "string", "minLength": 1},
+        "license_decision": {"enum": ["APPROVED_REDISTRIBUTABLE", "APPROVED_REBUILD_ONLY", "QUARANTINED"]},
+        "license_manifest_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"},
+        "terminal_inputs_used": {"const": false}
+      },
+      "additionalProperties": true
+    },
+    "rendering": {"type": "object", "required": ["parameters", "font_sources", "normalized_pixel_sha256"]},
+    "payloads": {"type": "object", "required": ["image_path", "image_sha256", "record_sha256"]},
+    "gold": {"$ref": "#/$defs/canonicalTable"},
+    "prior": {"$ref": "#/$defs/canonicalTable"},
+    "views": {
+      "type": "object",
+      "required": ["gate", "explicit", "lora"],
+      "properties": {
+        "gate": {"type": "object", "required": ["label", "false_edit_cost", "missed_edit_cost"]},
+        "explicit": {"type": "object", "required": ["prior_state_sha256", "order_invariant_difference", "raw_text_frozen"]},
+        "lora": {"type": "object", "required": ["complete_canonical_target", "table_only"]}
+      },
+      "additionalProperties": false
+    },
+    "audits": {
+      "type": "object",
+      "required": ["schema_valid", "canonical_legal", "prior_canonical_legal", "grid_complete", "prior_grid_complete", "gold_recompiled", "token_ownership_valid", "geometry_valid", "semantic_replay_valid", "normalized_pixel_replay_valid", "cross_role_overlap_clear"]
+    }
+  },
+  "$defs": {
+    "canonicalTable": {
+      "type": "object",
+      "required": ["rows", "columns", "cells", "tokens", "semantic_state_sha256", "full_state_sha256"],
+      "properties": {
+        "rows": {"type": "integer", "minimum": 1},
+        "columns": {"type": "integer", "minimum": 1},
+        "cells": {"type": "array", "minItems": 1},
+        "tokens": {"type": "array"},
+        "semantic_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"},
+        "full_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}
+      },
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": false
+}