diff --git a/.gitignore b/.gitignore index 2b3e3aa..8247e23 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ .idea/ .vscode/ .venv/ +.venv*/ __pycache__/ *.py[cod] *.egg-info/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index acc55f9..d373c42 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,12 +34,18 @@ evaluation artifacts use calendar versioning without a `v` prefix. | Dataset release | `data-2026.08.12` | `data-2026.08.12.1` | | Schema release | `canonical-table-record-2026.08.12` | `canonical-table-record-2026.08.12.1` | | Experiment | `experiment-2026.08.12-` | append `.1` when needed | +| Data smoke | `data-smoke-2026.08.12` | `data-smoke-2026.08.12.1` | Use the most precise meaningful date. The identifier does not claim that every artifact was created at exactly the same time; its purpose is to distinguish research states reproducibly. Upstream packages, datasets, and benchmarks keep their official external version names. +Project-authored research documents other than conventional repository files +such as `README.md`, `CONTRIBUTING.md`, and `LICENSE` include their CalVer in +the filename. Do not rename upstream action versions, benchmark releases, +pretrained-model names, or historical source-dataset identifiers. + ## Branches and pull requests Use a focused branch from the current default branch: diff --git a/README.md b/README.md index fb7ca2b..641eb75 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,14 @@ OmniDocBench is used as one document-parsing evaluation protocol. The methods and infrastructure in this repository are designed around the broader research problem of table structure recognition. -## Research snapshot 2026.08.12 +## Research snapshot 2026.08.12.1 -The `2026.08.12` snapshot establishes the shared representation and evaluation -foundation for two independent modeling tracks. It includes Canonical Table -normalization, order-invariant topology targets, candidate-integrity checks, -table-only model interfaces, and synthetic data-free tests. Model checkpoints -and dataset payloads are maintained outside this repository. +The `2026.08.12.1` snapshot establishes the shared representation, evaluation, +and synthetic-corpus specification for two independent modeling tracks. It +includes Canonical Table normalization, order-invariant topology targets, +candidate-integrity checks, table-only model interfaces, a frozen 256-record +generator-smoke design, and data-free regression tests. Model checkpoints and +dataset payloads are maintained outside this repository. Project-authored releases follow calendar versioning: @@ -54,7 +55,7 @@ separate signals so that a structural hypothesis can be inspected and replayed. The current repository includes the candidate representation, reversible interface, and validation primitives needed by the model. -See [Explicit Layout Transformer](docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md). +See [Explicit Layout Transformer](docs/methods/EXPLICIT_LAYOUT_TRANSFORMER_2026.08.12.1.md). ### LoRA Table Model @@ -96,11 +97,11 @@ The common foundation provides: borderless-table-structuring-lab/ ├── dataset/ # External dataset registry; no payloads ├── docs/ -│ ├── methods/ # Research-track formulations -│ ├── DATA_GOVERNANCE.md -│ ├── DATASET_STORAGE_AND_SHARING.md -│ ├── REPRODUCIBILITY.md -│ └── SYNTHETIC_CORPUS_SPECIFICATION.md +│ ├── corpus/ # CalVer data specifications and policies +│ ├── experiment-records/ # Sealed public research evidence +│ ├── methods/ # CalVer research-track formulations +│ └── REPRODUCIBILITY_2026.08.12.1.md +├── configs/ # Calendar-versioned generation parameters ├── schemas/ # Calendar-versioned record schemas ├── src/borderless_table_structuring/ │ ├── canonical.py # Canonical table normalization @@ -142,8 +143,8 @@ compression, blur, background, and scanning artifacts. Dataset roles are assigned by document, template, content, renderer, and seed families before rendering. Exact and near-duplicate audits operate on images, text, normalized structure, geometry, and provenance. See the -[synthetic corpus specification](docs/SYNTHETIC_CORPUS_SPECIFICATION.md) and -[data governance guide](docs/DATA_GOVERNANCE.md). +[synthetic data specification](docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md) +and [data governance guide](docs/corpus/DATA_GOVERNANCE_2026.08.12.1.md). ## Installation @@ -172,8 +173,8 @@ and complete failure accounting. Dataset payloads and model weights are stored outside Git; this repository contains the code, schemas, manifests, and documentation needed to reproduce them. -For details, see [Reproducibility](docs/REPRODUCIBILITY.md) and -[Dataset Storage and Sharing](docs/DATASET_STORAGE_AND_SHARING.md). +For details, see [Reproducibility](docs/REPRODUCIBILITY_2026.08.12.1.md) and +[Dataset Storage and Sharing](docs/corpus/DATASET_STORAGE_AND_SHARING_2026.08.12.1.md). ## Collaboration diff --git a/configs/generation_parameters_2026.08.12.1.json b/configs/generation_parameters_2026.08.12.1.json new file mode 100644 index 0000000..bc68a20 --- /dev/null +++ b/configs/generation_parameters_2026.08.12.1.json @@ -0,0 +1,75 @@ +{ + "configuration_release": "2026.08.12.1", + "dataset_release": "data-smoke-2026.08.12.1", + "schema_release": "synthetic-table-record-2026.08.12.1", + "seed_namespace": "borderless-table-structuring-data-smoke-2026.08.12.1", + "seed_start": 202608120100000, + "requested_records": 256, + "allow_failed_seed_replacement": false, + "gate_costs": { + "false_edit": 4.0, + "missed_edit": 1.0, + "uncertain_action": "KEEP" + }, + "categories": { + "exact_keep": {"count": 64, "gate_label": "KEEP"}, + "hard_keep": {"count": 64, "gate_label": "KEEP"}, + "single_minimal_edit": {"count": 64, "gate_label": "EDIT"}, + "complex_correction": {"count": 64, "gate_label": "EDIT"} + }, + "roles_per_category": { + "train": 44, + "development": 12, + "holdout": 8 + }, + "counterfactual_pairs": { + "minimum_pairs": 64, + "same_role_required": true, + "shared_fields": [ + "content_family_id", + "template_family_id", + "font_family_id", + "renderer_family_id", + "base_degradation" + ] + }, + "structure": { + "rows": {"minimum": 2, "maximum": 28}, + "columns": {"minimum": 2, "maximum": 14}, + "header_depth": {"minimum": 0, "maximum": 4}, + "span_fraction": {"minimum": 0.0, "maximum": 0.45}, + "empty_cell_fraction": {"minimum": 0.0, "maximum": 0.35}, + "maximum_cells": 240 + }, + "rendering": { + "canvas_width_px": [960, 1280, 1600, 2048], + "dpi": [96, 150, 200, 300], + "font_size_px": {"minimum": 12, "maximum": 32}, + "cell_padding_px": {"minimum": 4, "maximum": 18}, + "rotation_degrees": {"minimum": -4.0, "maximum": 4.0}, + "border_alpha": {"minimum": 0.0, "maximum": 1.0}, + "jpeg_quality": {"minimum": 55, "maximum": 100}, + "gaussian_blur_radius": {"minimum": 0.0, "maximum": 1.8}, + "noise_sigma": {"minimum": 0.0, "maximum": 12.0} + }, + "corruptions": { + "single_minimal_edit_exactly_one": true, + "complex_correction_minimum_operations": 2, + "complex_correction_maximum_operations": 4, + "allowed": [ + "extra_split", + "missing_split", + "incorrect_merge", + "missing_merge", + "span_extent_error", + "token_ownership_error", + "geometry_inconsistency", + "localized_grid_coverage_error" + ] + }, + "determinism": { + "semantic_replay_runs": 2, + "normalized_pixel_replay_runs": 2, + "sha256_algorithm": "sha256" + } +} diff --git a/dataset/README.md b/dataset/README.md index 43b0ccb..ab9a368 100644 --- a/dataset/README.md +++ b/dataset/README.md @@ -6,13 +6,13 @@ dataset references used by reproducible experiments. ## Active dataset entry template ```yaml -dataset_name: borderless-table-structuring-data +dataset_name: data-2026.08.12.1 service: private-hugging-face-or-approved-object-store repository: DearKarl/borderless-table-structuring-data revision: null root_manifest_sha256: null -schema_release: null -research_snapshot: null +schema_release: synthetic-table-record-2026.08.12.1 +research_snapshot: 2026.08.12.1 roles_available: [] license_manifest_sha256: null overlap_audit_sha256: null diff --git a/docs/REPRODUCIBILITY.md b/docs/REPRODUCIBILITY_2026.08.12.1.md similarity index 95% rename from docs/REPRODUCIBILITY.md rename to docs/REPRODUCIBILITY_2026.08.12.1.md index e635ac3..fa82cdc 100644 --- a/docs/REPRODUCIBILITY.md +++ b/docs/REPRODUCIBILITY_2026.08.12.1.md @@ -1,4 +1,4 @@ -# Reproducibility +# Reproducibility 2026.08.12.1 ## Immutable inputs @@ -17,6 +17,7 @@ Never reuse an output directory. A run writes to a new directory containing: - `quarantine.jsonl`; - `SHA256SUMS`; - `EXPERIMENT_RECORD.md`; +- `EVIDENCE_CARD.md`; - an immutable checksum manifest. ## Determinism diff --git a/docs/SYNTHETIC_CORPUS_SPECIFICATION.md b/docs/SYNTHETIC_CORPUS_SPECIFICATION.md deleted file mode 100644 index e8985cf..0000000 --- a/docs/SYNTHETIC_CORPUS_SPECIFICATION.md +++ /dev/null @@ -1,80 +0,0 @@ -# Synthetic Corpus Specification - -Research snapshot: `2026.08.12` - -Status: `RESEARCH_DRAFT` - -## Primary record contract - -Every record contains a source-traceable rendered table, direct Canonical Table -Gold, OCR token sidecar, physical geometry, a Raw-like prior, and an -order-invariant structural difference. Ordered action programs may be retained -as replay evidence but are not primary supervision. - -Required identity fields: - -- `sample_id`; -- `document_cluster_id`; -- `template_family_id`; -- `content_family_id`; -- `renderer_family_id`; -- `generation_seed`; -- `role`; -- `schema_release`. - -Required provenance fields: - -- generator and calendar release; -- source and license decision; -- font sources; -- render and degradation parameters; -- record and payload SHA256 values; -- explicit confirmation that terminal inputs were not used. - -## Phenomenon coverage matrix - -| Family | Required variants | Safety counterexample | -|---|---|---| -| Headers | single, multi-level, irregular | correct Raw structure requiring KEEP | -| Spans | row, column, mixed 2D | visually aligned cells that must not merge | -| Corrections | split-only, merge-only, joint | zero-edit identity | -| Borders | weak, missing, partial, low contrast | visible guides that are not cell borders | -| Empty cells | empty, near-empty, whitespace | meaningful sparse text | -| Text | dense, small, long, multiline, mixed script | Raw OCR text that must remain frozen | -| Formulas | formula-only, mixed formula/text | ordinary symbols that are not formulas | -| Imaging | rotation, blur, compression, scan noise | clean high-resolution render | -| Local errors | one minimal reversible topology error | broad edits prohibited | - -## Distribution principles - -- KEEP examples must be substantial enough that editing is not the default - prior. -- Difficulty is balanced by template family rather than post-hoc metric - selection. -- Correction types are sampled before rendering. -- Degradation parameters are sampled independently within preregistered ranges. -- No distribution parameter is calibrated using terminal scores. - -Exact numerical ranges and requested counts must be frozen in a separate -preregistration before the bounded generator smoke. - -## Split policy - -Assign document, template, content, renderer, and seed families to one role -before rendering. No template family crosses train, development, holdout, or -terminal roles. - -## Acceptance tests - -- Canonical legality and complete grid coverage. -- Direct Gold recompilation. -- Complete, unique OCR token ownership. -- Finite and non-degenerate physical geometry. -- Deterministic semantic replay. -- Source and license completeness. -- Exact, perceptual, structure, text, source, and geometry overlap audit. -- Complete generated, passed, quarantined, and failed counts. - -An unresolved overlap, license ambiguity, terminal-data risk, missing -geometry, token loss, illegal topology, or nondeterministic replay is a hard -stop. diff --git a/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.1.md b/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.1.md new file mode 100644 index 0000000..3d83a67 --- /dev/null +++ b/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.1.md @@ -0,0 +1,63 @@ +# Acceptance Criteria 2026.08.12.1 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +## Preregistered smoke gate + +`data-smoke-2026.08.12.1` passes only when all criteria below pass. No failed +seed may be silently replaced, no category quota may be rebalanced after +generation, and no model performance metric may be used to select records. + +| Criterion | Required result | Failure action | +|---|---:|---| +| Requested records | 256 | Stop | +| Generated records | 256 | Stop | +| Accepted records | 256 | Stop | +| Exact KEEP / Hard KEEP / Single / Complex | 64 / 64 / 64 / 64 | Stop | +| Counterfactual pairs | at least 64 complete same-role pairs | Stop | +| Schema validation | 256/256 | Stop | +| Canonical legality | 256/256 | Stop | +| Complete grid coverage | 256/256 | Stop | +| Gold recompilation | 256/256 semantic identity | Stop | +| Unique OCR-token ownership | 100% of owned tokens | Stop | +| Finite complete geometry | 256/256 | Stop | +| KEEP semantic identity | 128/128 | Stop | +| EDIT replay to Gold | 128/128 | Stop | +| Explicit KEEP empty diff | 128/128 | Stop | +| LoRA target completeness | 256/256 | Stop | +| Semantic replay | identical across 2 runs | Stop | +| Normalized-pixel replay | identical across 2 runs | Stop | +| Source and license inventory | complete and approved | Stop | +| Cross-role exact overlap | 0 | Stop | +| Cross-role unresolved near overlap | 0 | Stop | +| Terminal-input usage | false for every record | Stop | +| Complete count accounting | requested/generated/accepted/quarantined/failed | Stop | + +## Validation reason codes + +Every failure uses a stable reason code, including: + +- `SCHEMA_INVALID`; +- `CANONICAL_ILLEGAL`; +- `GRID_INCOMPLETE`; +- `GOLD_RECOMPILE_MISMATCH`; +- `TOKEN_OWNERSHIP_INVALID`; +- `GEOMETRY_INVALID`; +- `KEEP_NOT_IDENTITY`; +- `EDIT_NOT_REPLAYABLE`; +- `NONDETERMINISTIC_SEMANTICS`; +- `NONDETERMINISTIC_PIXELS`; +- `SOURCE_OR_LICENSE_INCOMPLETE`; +- `CROSS_ROLE_EXACT_OVERLAP`; +- `CROSS_ROLE_NEAR_OVERLAP`; +- `TERMINAL_DERIVATION_RISK`. + +Quarantine manifests preserve every failed record identifier, frozen seed, +reason code, and diagnostic hash. Visual examples are QA evidence only. + +## Post-pass boundary + +A sealed pass authorizes a new non-overwriting shared-corpus build with frozen +distributions. It does not authorize Explicit or LoRA training. Corpus scale is +chosen from measured generation cost, coverage, and quarantine rate, not from +current official benchmark scores. diff --git a/docs/COLLABORATOR_HANDOFF.md b/docs/corpus/COLLABORATOR_HANDOFF_2026.08.12.1.md similarity index 83% rename from docs/COLLABORATOR_HANDOFF.md rename to docs/corpus/COLLABORATOR_HANDOFF_2026.08.12.1.md index 38a3cd7..aa8ed6f 100644 --- a/docs/COLLABORATOR_HANDOFF.md +++ b/docs/corpus/COLLABORATOR_HANDOFF_2026.08.12.1.md @@ -1,4 +1,4 @@ -# Collaborator Handoff +# Collaborator Handoff 2026.08.12.1 ## Goal @@ -23,6 +23,13 @@ Deliverables: - exact and perceptual duplicate reports; - an English experiment record. +The frozen starting point is the +[`2026.08.12.1` synthetic-data specification](SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md), +its [coverage matrix](COVERAGE_MATRIX_2026.08.12.1.csv), and the +[`2026.08.12.1` generation parameters](../../configs/generation_parameters_2026.08.12.1.json). +Changes require a new CalVer revision; collaborators must not edit the frozen +files in place. + ## Other work packages ### Coverage specification diff --git a/docs/corpus/COVERAGE_MATRIX_2026.08.12.1.csv b/docs/corpus/COVERAGE_MATRIX_2026.08.12.1.csv new file mode 100644 index 0000000..fff5425 --- /dev/null +++ b/docs/corpus/COVERAGE_MATRIX_2026.08.12.1.csv @@ -0,0 +1,30 @@ +category,gate_label,phenomenon,requested_count,train_count,development_count,holdout_count,counterfactual_pair_required,primary_risk +exact_keep,KEEP,simple_regular,12,8,2,2,false,false_edit +exact_keep,KEEP,hierarchical_header,12,8,2,2,false,false_edit +exact_keep,KEEP,row_or_column_span,12,8,2,2,false,false_edit +exact_keep,KEEP,empty_or_sparse_cells,8,6,2,0,false,false_edit +exact_keep,KEEP,dense_or_multiline_text,8,6,2,0,false,text_loss +exact_keep,KEEP,mixed_script_or_formula,6,4,2,0,false,text_loss +exact_keep,KEEP,clean_high_resolution_control,6,4,0,2,false,false_edit +hard_keep,KEEP,multilevel_irregular_header,12,8,2,2,true,false_edit +hard_keep,KEEP,mixed_two_dimensional_spans,12,8,2,2,true,false_edit +hard_keep,KEEP,weak_missing_or_partial_borders,12,8,2,2,true,false_edit +hard_keep,KEEP,empty_near_empty_or_sparse,8,6,2,0,true,false_edit +hard_keep,KEEP,dense_small_long_or_multiline_text,8,6,2,0,true,text_loss +hard_keep,KEEP,rotation_blur_compression_or_noise,8,4,2,2,true,false_edit +hard_keep,KEEP,color_background_or_alignment_trap,4,4,0,0,true,false_edit +single_minimal_edit,EDIT,extra_split,12,8,2,2,true,cell_inflation +single_minimal_edit,EDIT,missing_split,10,6,2,2,true,under_segmentation +single_minimal_edit,EDIT,incorrect_merge,10,6,2,2,true,content_reassignment +single_minimal_edit,EDIT,missing_merge,10,6,2,2,true,over_segmentation +single_minimal_edit,EDIT,span_extent_error,8,6,2,0,true,topology_error +single_minimal_edit,EDIT,token_ownership_error,6,4,2,0,true,text_loss +single_minimal_edit,EDIT,geometry_inconsistency,4,4,0,0,true,location_error +single_minimal_edit,EDIT,localized_grid_coverage_error,4,4,0,0,true,illegal_grid +complex_correction,EDIT,multilevel_header_with_spans,12,8,2,2,false,topology_error +complex_correction,EDIT,joint_split_and_merge,12,8,2,2,false,cell_inflation +complex_correction,EDIT,weak_border_with_topology_error,10,6,2,2,false,missed_edit +complex_correction,EDIT,empty_cells_with_spans,8,6,2,0,false,ownership_error +complex_correction,EDIT,dense_multiline_with_topology_error,8,6,2,0,false,text_loss +complex_correction,EDIT,mixed_script_formula_with_topology_error,6,4,2,0,false,text_loss +complex_correction,EDIT,rotation_noise_with_topology_error,8,6,0,2,false,geometry_error diff --git a/docs/DATASET_STORAGE_AND_SHARING.md b/docs/corpus/DATASET_STORAGE_AND_SHARING_2026.08.12.1.md similarity index 98% rename from docs/DATASET_STORAGE_AND_SHARING.md rename to docs/corpus/DATASET_STORAGE_AND_SHARING_2026.08.12.1.md index 1c470b3..ca0fddb 100644 --- a/docs/DATASET_STORAGE_AND_SHARING.md +++ b/docs/corpus/DATASET_STORAGE_AND_SHARING_2026.08.12.1.md @@ -1,4 +1,4 @@ -# Dataset Storage and Sharing +# Dataset Storage and Sharing 2026.08.12.1 ## Recommended architecture diff --git a/docs/DATA_GOVERNANCE.md b/docs/corpus/DATA_GOVERNANCE_2026.08.12.1.md similarity index 92% rename from docs/DATA_GOVERNANCE.md rename to docs/corpus/DATA_GOVERNANCE_2026.08.12.1.md index 5a41645..0cee62a 100644 --- a/docs/DATA_GOVERNANCE.md +++ b/docs/corpus/DATA_GOVERNANCE_2026.08.12.1.md @@ -1,4 +1,4 @@ -# Data Governance +# Data Governance 2026.08.12.1 ## Roles @@ -6,6 +6,9 @@ Every record has exactly one immutable role: `train`, `development`, `holdout`, or `terminal`. Roles are assigned before rendering and before any model or metric is run. +The active smoke allocation and role-level counts are frozen in +[`SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md`](SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md). + ## Required provenance Every generated or newly sourced record must include: diff --git a/docs/corpus/LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md b/docs/corpus/LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md new file mode 100644 index 0000000..4269e83 --- /dev/null +++ b/docs/corpus/LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md @@ -0,0 +1,72 @@ +# License and Source Policy 2026.08.12.1 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +## Scope + +This policy governs every content item, font, texture, rendering asset, source +table, prior parser, and external dataset referenced by +`data-smoke-2026.08.12.1` and later corpus builds derived from its generator. + +## Default source policy + +The smoke uses newly authored synthetic content. It does not ingest evaluation +pages, scraped documents, customer records, or third-party table annotations. +Names, values, formulas, and prose are generated from project-authored grammar +and deterministic seed tables. + +## Allowed source decisions + +Every source receives one decision: + +- `APPROVED_REDISTRIBUTABLE`: redistribution and derivative use are compatible + with the intended research release. +- `APPROVED_REBUILD_ONLY`: local processing is allowed, but payloads may not be + redistributed; only compilers, acquisition instructions, and hashes may be + shared. +- `QUARANTINED`: rights, attribution, origin, or downstream-use terms are + incomplete, incompatible, or disputed. + +Private access is not evidence of redistribution permission. + +## Required inventory fields + +The machine-readable source inventory records: + +- source identifier, title, owner, and acquisition URL or authored origin; +- acquisition date and immutable revision where available; +- license name, version, URL, and license-text SHA256; +- redistribution, modification, derivative, research, and commercial-use + decisions; +- attribution and notice requirements; +- reviewer, decision date, decision, and rationale; +- payload or source-manifest SHA256. + +## Font and rendering assets + +The initial smoke permits project-authored assets, public-domain assets, and +fonts with explicit redistribution and embedding rights, such as compatible +SIL Open Font License or Apache License releases. Each font file is pinned by +family, source URL, license, revision, and SHA256. System fonts with unclear +redistribution rights may be used only in `APPROVED_REBUILD_ONLY` outputs. + +## Benchmark exclusion + +Current OmniDocBench and Customer50 images, crops, text, coordinates, HTML, +LaTeX, identifiers, metadata, annotations, embeddings, or transformations are +prohibited. Public benchmark attribute names may inform coverage categories; +they may not provide record content. + +## External dataset rule + +An external dataset may enter a later dated corpus only through a new source +decision, an isolation audit, and a dated specification revision. Dataset +licenses remain independent of code-repository licensing. A noncommercial +benchmark cannot be silently converted into a redistributable or commercial +training source. + +## Hard stops + +Generation stops when a required license file, source revision, attribution, +permission decision, or hash is missing. Failed records remain in the source +quarantine manifest with a reason code; they are not replaced silently. diff --git a/docs/corpus/SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md b/docs/corpus/SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md new file mode 100644 index 0000000..50d297f --- /dev/null +++ b/docs/corpus/SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md @@ -0,0 +1,66 @@ +# Split and Isolation Policy 2026.08.12.1 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +## Role assignment + +Roles are assigned before rendering, corruption, model inference, or metric +calculation. The smoke contains: + +| Role | Records per category | Total records | Permitted use | +|---|---:|---:|---| +| train | 44 | 176 | Generator and future model fitting after authorization | +| development | 12 | 48 | Nonterminal validation and bounded selection | +| holdout | 8 | 32 | One-shot bounded evidence after configuration freeze | + +The smoke creates no terminal role. A later terminal role requires a separately +sealed seed range and one-shot protocol after all generator parameters are +frozen. + +## Family-level grouping + +The following identifiers must each map to exactly one role: + +- `document_cluster_id`; +- `template_family_id`; +- `content_family_id`; +- `renderer_family_id`; +- `font_family_id` when font appearance is part of a paired design; +- `counterfactual_pair_id`; +- `source_family_id`; +- generation-seed family. + +Counterfactual members always remain in the same role. A role-assignment file +is produced before any image exists and is frozen by SHA256. + +## Overlap audits + +Every build runs these cross-role audits: + +1. payload SHA256 and normalized-pixel SHA256 equality; +2. perceptual-image similarity under a preregistered threshold; +3. normalized Canonical Table topology equality; +4. normalized text-content equality and high-overlap shingles; +5. document, source, template, renderer, font, and pair-family equality; +6. quantized geometry-signature equality; +7. composite near-duplicate linkage across two or more weak signals. + +Exact overlap must be zero. A near-duplicate candidate is quarantined until an +English review record explains and resolves it. No unresolved cross-role link +is permitted. + +## Benchmark and collaborator isolation + +Current OmniDocBench and Customer50 artifacts are not audit inputs because they +are prohibited generator sources. A separate restricted environment may later +run one-way contamination checks that emit only aggregate overlap status and +hashes; it may not expose terminal content to generator authors. + +Collaborators receive only the roles and payloads necessary for their assigned +work. Anyone optimizing a generator or model does not receive terminal labels. + +## Non-overwrite policy + +Each build uses a new CalVer path. `data-smoke-2026.08.12.1` may never be +overwritten. Any correction creates `data-smoke-2026.08.12.2` or a later dated +identifier with a new manifest and Evidence Card. diff --git a/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md b/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md new file mode 100644 index 0000000..120c9a6 --- /dev/null +++ b/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md @@ -0,0 +1,160 @@ +# Synthetic Data Specification 2026.08.12.1 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +Dataset identifier: `data-smoke-2026.08.12.1` + +Schema identifier: `synthetic-table-record-2026.08.12.1` + +## Research question + +Can a shared synthetic corpus teach a table model to preserve an already +correct prior and to correct only deterministic structural defects, without +encoding one arbitrary sequence of split and merge actions as the answer? + +The first bounded test is a data-generation and validation smoke. It does not +train a model and does not support a performance claim. + +## Evidence basis + +The design follows four observations from primary research and official +benchmark materials: + +1. PubTables-1M identifies inconsistent over-segmentation as a major source of + table ground-truth noise and uses canonicalization to remove representational + ambiguity. +2. GriTS evaluates topology, location, and content as separate matrix-level + properties, so a legal serialization alone is not sufficient evidence of a + good table. +3. OmniDocBench exposes table attributes such as border condition, merged + cells, formulas, color, and rotation. These public categories define + phenomenon coverage, not page-level source material. +4. Synthetic-table research reports benefits from controlling structural + complexity, especially spanning-cell distributions, while preserving an + independently sourced evaluation boundary. + +Primary references: + +- [PubTables-1M](https://arxiv.org/abs/2110.00061) +- [GriTS](https://arxiv.org/abs/2203.12555) +- [OmniDocBench official repository](https://github.com/opendatalab/OmniDocBench) +- [Synthesizing Realistic Data for Table Recognition](https://arxiv.org/abs/2404.11100) + +## Supervision contract + +Every attempted record receives exactly one gate label: + +- `KEEP`: the Raw-like prior and Gold are semantically equivalent in topology, + token ownership, and valid geometry. Serialization differences do not turn a + record into an edit. +- `EDIT`: the prior contains a deterministic, replayable structural difference + from Gold. +- `QUARANTINE`: source rights, semantics, geometry, ownership, determinism, or + isolation are ambiguous or invalid. + +An OCR transcription error does not by itself authorize an Explicit topology +edit. The Explicit target is an order-invariant structural difference and is +empty for `KEEP`. The LoRA target is one complete Canonical Table state and is +semantically equivalent to the prior for `KEEP`. + +## Gate and editor boundary + +The gate decides `KEEP` or `EDIT` before a route-specific candidate is used. +The editor is invoked only for `EDIT`. Both routes share canonical legality, +OCR-token preservation, geometry completeness, table-only assembly, and exact +prior rollback checks. A rejected or uncertain candidate leaves the prior +unchanged. + +The initial decision cost assigns a false edit four times the cost of a missed +edit. This value is preregistered for the smoke and may be changed only by a +new nonterminal single-variable experiment. Current official benchmark scores +may not calibrate it. + +## Bounded smoke composition + +The smoke contains exactly 256 accepted records if and only if every frozen +seed passes. There is no silent replacement of a failed seed. + +| Category | Label | Count | Purpose | +|---|---|---:|---| +| Exact KEEP | KEEP | 64 | Byte- or state-equivalent correct priors | +| Hard KEEP | KEEP | 64 | Visually difficult but semantically correct priors | +| Single minimal edit | EDIT | 64 | One controlled reversible topology defect | +| Complex correction | EDIT | 64 | Multiple interacting but deterministic defects | + +At least 64 counterfactual pairs are required. Each pair shares authored +content, template family, font family, renderer family, and base degradation. +One member remains correct (`KEEP`); the other receives a frozen controlled +structural corruption (`EDIT`). The pair must not cross data roles. + +The four smoke categories allocate `44/12/8` records to +`train/development/holdout`, respectively. A terminal role is not created in +the smoke. A later corpus may use terminal records only after the generator, +schema, and distribution are frozen and under a separate one-shot protocol. + +## Prior construction + +Two prior families are allowed: + +1. `controlled_identity_or_error`: start from newly authored Canonical Gold; + use identity for KEEP or a deterministic corruption operator for EDIT. +2. `approved_real_parser`: run a frozen parser on a newly authored or approved + nonterminal image and retain complete parser provenance. This family is + initially limited to KEEP-oriented evidence until its licensing and output + determinism are separately sealed. + +No current OmniDocBench or Customer50 page, crop, text, coordinate, annotation, +embedding, identifier, or transformed derivative may be used as a source or +retrieval query. + +## Record views + +Each accepted record exposes three synchronized views: + +- Gate view: prior evidence and the `KEEP` or `EDIT` target. +- Explicit view: the minimal order-invariant topology difference; empty for + KEEP, with prior text frozen. +- LoRA view: a complete Canonical Table target with topology, geometry, content, + and ownership. + +All views bind to the same image, prior-state hash, Gold-state hash, split role, +and provenance record. + +## Deterministic generation + +The generator samples structure before rendering. Template, content, font, +renderer, corruption, and degradation parameters are derived from the frozen +seed namespace in +[`generation_parameters_2026.08.12.1.json`](../../configs/generation_parameters_2026.08.12.1.json). + +The same inputs must reproduce identical normalized semantic records and +normalized pixels. Encoders that cannot guarantee byte-identical image files +must record both file SHA256 and normalized-pixel SHA256. + +## Split and isolation + +Document, template, content, renderer, and counterfactual-pair families are +assigned to one role before rendering. The policy is defined in +[`SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md`](SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md). + +Exact file, normalized pixel, perceptual image, normalized structure, +normalized text, source, and geometry-signature audits are mandatory. Any +unresolved cross-role overlap is a hard stop. + +## Rights and provenance + +The smoke uses newly authored synthetic content and fonts/assets with explicit +compatible rights. Every source and font decision follows +[`LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md`](LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md). +Unknown, non-redistributable, or incompatible material is quarantined before +generation. + +## Acceptance and escalation + +The smoke passes only when every criterion in +[`ACCEPTANCE_CRITERIA_2026.08.12.1.md`](ACCEPTANCE_CRITERIA_2026.08.12.1.md) +passes and the Evidence Card is sealed. A pass authorizes a new, +non-overwriting shared-corpus build. A failure stops bulk generation and +requires a new dated specification revision. + +Neither a smoke pass nor a corpus build authorizes model training. diff --git a/docs/experiment-records/EVIDENCE_CARD_SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md b/docs/experiment-records/EVIDENCE_CARD_SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md new file mode 100644 index 0000000..51053d4 --- /dev/null +++ b/docs/experiment-records/EVIDENCE_CARD_SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md @@ -0,0 +1,80 @@ +# Evidence Card: Synthetic Data Specification 2026.08.12.1 + +Status: `PASS_SEALED_FOR_BOUNDED_GENERATOR_IMPLEMENTATION` + +## Objective + +Freeze an English, CalVer-addressed specification for a shared synthetic table +corpus before any batch generation or model training. The specification must +make KEEP supervision, deterministic structural correction, provenance, +license decisions, split isolation, and hard acceptance gates executable. + +## Authorization and boundary + +- Work type: documentation, schema, configuration, and preregistration. +- Output release: `2026.08.12.1`. +- Next bounded dataset: `data-smoke-2026.08.12.1`. +- Data generated in this stage: `0` records. +- Model training started: `false`. +- Current OmniDocBench or Customer50 contents used: `false`. +- Current official benchmark evaluation run: `false`. +- Historical sealed artifacts modified: `false`. + +## Frozen decisions + +1. Gate labels are `KEEP`, `EDIT`, and `QUARANTINE`. +2. Semantic equivalence is KEEP even when serialization differs. +3. Explicit supervision is an order-invariant topology difference with prior + text frozen; LoRA supervision is a complete Canonical Table state. +4. The 256-record smoke contains 64 Exact KEEP, 64 Hard KEEP, 64 Single + Minimal Edit, and 64 Complex Correction records. +5. The smoke split is 176 train, 48 development, and 32 holdout records; it has + no terminal role. +6. At least 64 same-role counterfactual pairs are required. +7. False edits cost four times missed edits, and uncertainty resolves to KEEP. +8. Failed seeds are not silently replaced. +9. All acceptance checks must pass before bulk generation. +10. Neither a data-smoke pass nor a corpus build authorizes model training. + +## Research basis + +- PubTables-1M motivates canonical ground truth and removal of inconsistent + over-segmentation. +- GriTS motivates separate topology, location, and content views. +- OmniDocBench official attributes motivate generic coverage categories without + supplying page-level training material. +- Synthetic table-recognition research motivates explicit control of structural + complexity and spanning-cell distributions. + +## Validation performed + +- All JSON files parsed successfully. +- Coverage matrix totals closed at `256/176/48/32` for + requested/train/development/holdout. +- Each of the four primary categories totaled 64 records. +- All local Markdown links resolved. +- Repository diff passed `git diff --check`. +- The data-free regression suite passed `18/18` tests under Python 3.12. +- Public-tree naming audit found no project-owned legacy dataset-name, + baseline-number, or internal contract-version identifiers. +- Upstream dependency and benchmark version names were preserved. + +## Frozen artifact hashes + +| Artifact | SHA256 | +|---|---| +| `configs/generation_parameters_2026.08.12.1.json` | `700c94b47a0f1e3e0e42c60e2eec6c33ff19c438f6afee047ffe49e80cf6d02d` | +| `docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md` | `430a6758f22043e647bb42d1bebaa23f99702b08ccf1e92ca022e92e00054bb4` | +| `docs/corpus/COVERAGE_MATRIX_2026.08.12.1.csv` | `833c750ddd72ec27b218ca96e3309f15d2b59f6d389e9af88d76df28e20d0795` | +| `docs/corpus/LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md` | `962840f33fdc4187ef9c1a8106b1d1efd7a00f8b12f5091db03ae2f0bf5e1dae` | +| `docs/corpus/SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md` | `1af384360ea2726558dfb9625eebbe647e4170b7b2abb4324b55f45bda763b97` | +| `docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.1.md` | `2a57ecc701ae1c77d9dcd8a51b70c8b4b8121cedb212c2e58d4631316cccbfae` | +| `schemas/synthetic_table_record_2026.08.12.1.json` | `b8ae27175fe6147a8d14c380f92cbb82d13cffa4673735aaba79182c1e17fcf3` | +| `schemas/source_license_record_2026.08.12.1.json` | `5f4194eab163e36b2fc1090e808d9baf48085317cca1a6a9c868266e96316203` | + +## Result + +The specification stage passes and authorizes implementation of the bounded +256-record generator smoke in a new non-overwriting path. Bulk corpus +generation remains conditional on a sealed smoke pass. Model training remains +unauthorized. diff --git a/docs/experiment-records/EVIDENCE_CARD_SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.sha256 b/docs/experiment-records/EVIDENCE_CARD_SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.sha256 new file mode 100644 index 0000000..18cbf32 --- /dev/null +++ b/docs/experiment-records/EVIDENCE_CARD_SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.sha256 @@ -0,0 +1 @@ +fef7930ff15d35e7b5810e734486d44b36db0afa6a16109e32150dd4e1848ef1 EVIDENCE_CARD_SYNTHETIC_DATA_SPECIFICATION_2026.08.12.1.md diff --git a/docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md b/docs/methods/EXPLICIT_LAYOUT_TRANSFORMER_2026.08.12.1.md similarity index 98% rename from docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md rename to docs/methods/EXPLICIT_LAYOUT_TRANSFORMER_2026.08.12.1.md index 0f34d0d..972dc64 100644 --- a/docs/methods/EXPLICIT_LAYOUT_TRANSFORMER.md +++ b/docs/methods/EXPLICIT_LAYOUT_TRANSFORMER_2026.08.12.1.md @@ -1,4 +1,4 @@ -# Explicit Layout Transformer +# Explicit Layout Transformer 2026.08.12.1 ## Research question diff --git a/pyproject.toml b/pyproject.toml index e61bec1..0685775 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "borderless-table-structuring-lab" -version = "2026.8.12" +version = "2026.8.12.1" description = "Research on canonical representations, explicit topology modeling, and generative adaptation for borderless table structure recognition." readme = "README.md" requires-python = ">=3.10" diff --git a/schemas/source_license_record_2026.08.12.1.json b/schemas/source_license_record_2026.08.12.1.json new file mode 100644 index 0000000..03768fc --- /dev/null +++ b/schemas/source_license_record_2026.08.12.1.json @@ -0,0 +1,67 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "borderless-table-structuring/source-license-record/2026.08.12.1", + "title": "Source License Record 2026.08.12.1", + "type": "object", + "required": [ + "schema_release", + "source_id", + "source_title", + "source_owner", + "acquisition_method", + "acquisition_date", + "source_revision", + "license", + "permissions", + "review", + "source_manifest_sha256" + ], + "properties": { + "schema_release": {"const": "source-license-record-2026.08.12.1"}, + "source_id": {"type": "string", "minLength": 1}, + "source_title": {"type": "string", "minLength": 1}, + "source_owner": {"type": "string", "minLength": 1}, + "acquisition_method": {"enum": ["PROJECT_AUTHORED", "DOWNLOAD", "LOCAL_REBUILD", "OTHER"]}, + "acquisition_url": {"type": ["string", "null"]}, + "acquisition_date": {"type": "string", "format": "date"}, + "source_revision": {"type": "string", "minLength": 1}, + "license": { + "type": "object", + "required": ["name", "version", "url", "text_sha256"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "url": {"type": ["string", "null"]}, + "text_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": false + }, + "permissions": { + "type": "object", + "required": ["redistribution", "modification", "derivatives", "research_use", "commercial_use", "attribution_required"], + "properties": { + "redistribution": {"type": "boolean"}, + "modification": {"type": "boolean"}, + "derivatives": {"type": "boolean"}, + "research_use": {"type": "boolean"}, + "commercial_use": {"type": ["boolean", "null"]}, + "attribution_required": {"type": "boolean"} + }, + "additionalProperties": false + }, + "attribution_notice": {"type": ["string", "null"]}, + "review": { + "type": "object", + "required": ["decision", "reviewer", "decision_date", "rationale"], + "properties": { + "decision": {"enum": ["APPROVED_REDISTRIBUTABLE", "APPROVED_REBUILD_ONLY", "QUARANTINED"]}, + "reviewer": {"type": "string", "minLength": 1}, + "decision_date": {"type": "string", "format": "date"}, + "rationale": {"type": "string", "minLength": 1} + }, + "additionalProperties": false + }, + "source_manifest_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": false +} diff --git a/schemas/synthetic_table_record_2026.08.12.1.json b/schemas/synthetic_table_record_2026.08.12.1.json new file mode 100644 index 0000000..869fbbb --- /dev/null +++ b/schemas/synthetic_table_record_2026.08.12.1.json @@ -0,0 +1,133 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "borderless-table-structuring/synthetic-table-record/2026.08.12.1", + "title": "Synthetic Table Record 2026.08.12.1", + "type": "object", + "required": [ + "schema_release", + "dataset_release", + "sample_id", + "role", + "gate_label", + "category", + "identity", + "provenance", + "rendering", + "payloads", + "gold", + "prior", + "views", + "audits" + ], + "properties": { + "schema_release": {"const": "synthetic-table-record-2026.08.12.1"}, + "dataset_release": {"const": "data-smoke-2026.08.12.1"}, + "sample_id": {"type": "string", "pattern": "^data-smoke-2026\\.08\\.12\\.1-[0-9]{6}$"}, + "role": {"enum": ["train", "development", "holdout"]}, + "gate_label": {"enum": ["KEEP", "EDIT", "QUARANTINE"]}, + "category": { + "enum": ["exact_keep", "hard_keep", "single_minimal_edit", "complex_correction"] + }, + "identity": { + "type": "object", + "required": [ + "generation_seed", + "document_cluster_id", + "source_family_id", + "template_family_id", + "content_family_id", + "renderer_family_id", + "font_family_id", + "counterfactual_pair_id" + ], + "properties": { + "generation_seed": {"type": "integer", "minimum": 0}, + "document_cluster_id": {"type": "string", "minLength": 1}, + "source_family_id": {"type": "string", "minLength": 1}, + "template_family_id": {"type": "string", "minLength": 1}, + "content_family_id": {"type": "string", "minLength": 1}, + "renderer_family_id": {"type": "string", "minLength": 1}, + "font_family_id": {"type": "string", "minLength": 1}, + "counterfactual_pair_id": {"type": ["string", "null"]} + }, + "additionalProperties": false + }, + "provenance": { + "type": "object", + "required": [ + "generator_release", + "source_id", + "license_decision", + "license_manifest_sha256", + "terminal_inputs_used" + ], + "properties": { + "generator_release": {"const": "2026.08.12.1"}, + "source_id": {"type": "string", "minLength": 1}, + "license_decision": {"enum": ["APPROVED_REDISTRIBUTABLE", "APPROVED_REBUILD_ONLY", "QUARANTINED"]}, + "license_manifest_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "terminal_inputs_used": {"const": false} + }, + "additionalProperties": true + }, + "rendering": { + "type": "object", + "required": ["parameters", "font_sources", "normalized_pixel_sha256"] + }, + "payloads": { + "type": "object", + "required": ["image_path", "image_sha256", "record_sha256"] + }, + "gold": {"$ref": "#/$defs/canonicalTable"}, + "prior": {"$ref": "#/$defs/canonicalTable"}, + "views": { + "type": "object", + "required": ["gate", "explicit", "lora"], + "properties": { + "gate": { + "type": "object", + "required": ["label", "false_edit_cost", "missed_edit_cost"] + }, + "explicit": { + "type": "object", + "required": ["prior_state_sha256", "order_invariant_difference", "raw_text_frozen"] + }, + "lora": { + "type": "object", + "required": ["complete_canonical_target", "table_only"] + } + }, + "additionalProperties": false + }, + "audits": { + "type": "object", + "required": [ + "schema_valid", + "canonical_legal", + "grid_complete", + "gold_recompiled", + "token_ownership_valid", + "geometry_valid", + "semantic_replay_valid", + "normalized_pixel_replay_valid", + "cross_role_overlap_clear" + ] + } + }, + "$defs": { + "canonicalTable": { + "type": "object", + "required": ["rows", "columns", "cells", "tokens", "semantic_state_sha256", "full_state_sha256"], + "properties": { + "rows": {"type": "integer", "minimum": 1}, + "columns": {"type": "integer", "minimum": 1}, + "cells": {"type": "array", "minItems": 1}, + "tokens": {"type": "array"}, + "semantic_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "full_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/src/borderless_table_structuring/__init__.py b/src/borderless_table_structuring/__init__.py index 3e9c8b3..c756aeb 100644 --- a/src/borderless_table_structuring/__init__.py +++ b/src/borderless_table_structuring/__init__.py @@ -1,5 +1,5 @@ """Research primitives for borderless table structure recognition.""" -__release__ = "2026.08.12" +__release__ = "2026.08.12.1" __all__ = ["__release__"]