diff --git a/README.md b/README.md index 89383d3..0a59629 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,9 @@ OmniDocBench is used as one document-parsing evaluation protocol. The methods and infrastructure in this repository are designed around the broader research problem of table structure recognition. -## Research snapshot 2026.08.12.2 +## Research snapshot 2026.08.12.4 -The `2026.08.12.2` snapshot establishes the shared representation, evaluation, +The `2026.08.12.4` snapshot establishes the shared representation, evaluation, and synthetic-corpus specification for two independent modeling tracks. It includes Canonical Table normalization, order-invariant topology targets, candidate-integrity checks, table-only model interfaces, a frozen 256-record @@ -38,7 +38,8 @@ dataset payloads are maintained outside this repository. Project-authored releases follow calendar versioning: - research snapshots: `2026.08.12`, `2026.09.03`, and so on; -- same-day revisions: `2026.08.12.1`, `2026.08.12.2`, and so on; +- same-day revisions: `2026.08.12.1`, `2026.08.12.2`, `2026.08.12.3`, + `2026.08.12.4`, and so on; - model artifacts: `explicit-2026.08.12` and `lora-2026.08.12`. External software and benchmark releases retain their original upstream names. @@ -103,6 +104,7 @@ borderless-table-structuring-lab/ │ └── REPRODUCIBILITY_2026.08.12.1.md ├── configs/ # Calendar-versioned generation parameters ├── schemas/ # Calendar-versioned record schemas +├── scripts/ # Calendar-versioned corpus entry points ├── src/borderless_table_structuring/ │ ├── canonical.py # Canonical table normalization │ ├── explicit.py # Public Explicit-route interface @@ -143,7 +145,7 @@ compression, blur, background, and scanning artifacts. Dataset roles are assigned by document, template, content, renderer, and seed families before rendering. Exact and near-duplicate audits operate on images, text, normalized structure, geometry, and provenance. See the -[synthetic data specification](docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.2.md) +[synthetic data specification](docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.4.md) and [data governance guide](docs/corpus/DATA_GOVERNANCE_2026.08.12.1.md). ## Installation @@ -165,6 +167,13 @@ Run the data-free test suite: pytest ``` +Generate the bounded synthetic-data smoke into a new external payload path: + +```bash +python scripts/generate_data_smoke_2026.08.12.4.py \ + --output /absolute/path/to/data-smoke-2026.08.12.4 +``` + ## Reproducible research Experiments record the code revision, schema release, immutable data revision, diff --git a/configs/generation_parameters_2026.08.12.3.json b/configs/generation_parameters_2026.08.12.3.json new file mode 100644 index 0000000..6dceedb --- /dev/null +++ b/configs/generation_parameters_2026.08.12.3.json @@ -0,0 +1,77 @@ +{ + "configuration_release": "2026.08.12.3", + "dataset_release": "data-smoke-2026.08.12.3", + "schema_release": "synthetic-table-record-2026.08.12.3", + "seed_namespace": "borderless-table-structuring-data-smoke-2026.08.12.3", + "seed_start": 202608120300000, + "requested_records": 256, + "allow_failed_seed_replacement": false, + "gate_costs": { + "false_edit": 4.0, + "missed_edit": 1.0, + "uncertain_action": "KEEP" + }, + "categories": { + "exact_keep": {"count": 64, "gate_label": "KEEP"}, + "hard_keep": {"count": 64, "gate_label": "KEEP"}, + "single_minimal_edit": {"count": 64, "gate_label": "EDIT"}, + "complex_correction": {"count": 64, "gate_label": "EDIT"} + }, + "roles_per_category": { + "train": 44, + "development": 12, + "holdout": 8 + }, + "counterfactual_pairs": { + "minimum_pairs": 64, + "same_role_required": true, + "shared_fields": [ + "content_family_id", + "template_family_id", + "font_family_id", + "renderer_family_id", + "base_degradation" + ] + }, + "structure": { + "rows": {"minimum": 2, "maximum": 28}, + "columns": {"minimum": 2, "maximum": 14}, + "header_depth": {"minimum": 0, "maximum": 4}, + "span_fraction": {"minimum": 0.0, "maximum": 0.45}, + "empty_cell_fraction": {"minimum": 0.0, "maximum": 0.35}, + "maximum_cells": 240 + }, + "rendering": { + "canvas_width_px": [960, 1280, 1600, 2048], + "dpi": [96, 150, 200, 300], + "font_size_px": {"minimum": 12, "maximum": 32}, + "cell_padding_px": {"minimum": 4, "maximum": 18}, + "rotation_degrees": {"minimum": -4.0, "maximum": 4.0}, + "border_alpha": {"minimum": 0.0, "maximum": 1.0}, + "jpeg_quality": {"minimum": 55, "maximum": 100}, + "gaussian_blur_radius": {"minimum": 0.0, "maximum": 1.8}, + "noise_sigma": {"minimum": 0.0, "maximum": 12.0} + }, + "corruptions": { + "single_minimal_edit_exactly_one": true, + "complex_correction_minimum_operations": 2, + "complex_correction_maximum_operations": 4, + "accepted_prior_must_remain_canonical_legal": true, + "invalid_grid_fixtures_are_test_only": true, + "allowed": [ + "extra_split", + "missing_split", + "incorrect_merge", + "missing_merge", + "span_extent_error", + "token_ownership_error", + "geometry_inconsistency", + "row_or_column_assignment_error" + ] + }, + "determinism": { + "semantic_replay_runs": 2, + "normalized_pixel_replay_runs": 2, + "sha256_algorithm": "sha256" + } +} diff --git a/configs/generation_parameters_2026.08.12.4.json b/configs/generation_parameters_2026.08.12.4.json new file mode 100644 index 0000000..41a6260 --- /dev/null +++ b/configs/generation_parameters_2026.08.12.4.json @@ -0,0 +1,77 @@ +{ + "configuration_release": "2026.08.12.4", + "dataset_release": "data-smoke-2026.08.12.4", + "schema_release": "synthetic-table-record-2026.08.12.4", + "seed_namespace": "borderless-table-structuring-data-smoke-2026.08.12.4", + "seed_start": 202608120400000, + "requested_records": 256, + "allow_failed_seed_replacement": false, + "gate_costs": { + "false_edit": 4.0, + "missed_edit": 1.0, + "uncertain_action": "KEEP" + }, + "categories": { + "exact_keep": {"count": 64, "gate_label": "KEEP"}, + "hard_keep": {"count": 64, "gate_label": "KEEP"}, + "single_minimal_edit": {"count": 64, "gate_label": "EDIT"}, + "complex_correction": {"count": 64, "gate_label": "EDIT"} + }, + "roles_per_category": { + "train": 44, + "development": 12, + "holdout": 8 + }, + "counterfactual_pairs": { + "minimum_pairs": 64, + "same_role_required": true, + "shared_fields": [ + "content_family_id", + "template_family_id", + "font_family_id", + "renderer_family_id", + "base_degradation" + ] + }, + "structure": { + "rows": {"minimum": 2, "maximum": 28}, + "columns": {"minimum": 2, "maximum": 14}, + "header_depth": {"minimum": 0, "maximum": 4}, + "span_fraction": {"minimum": 0.0, "maximum": 0.45}, + "empty_cell_fraction": {"minimum": 0.0, "maximum": 0.35}, + "maximum_cells": 240 + }, + "rendering": { + "canvas_width_px": [960, 1280, 1600, 2048], + "dpi": [96, 150, 200, 300], + "font_size_px": {"minimum": 12, "maximum": 32}, + "cell_padding_px": {"minimum": 4, "maximum": 18}, + "rotation_degrees": {"minimum": -4.0, "maximum": 4.0}, + "border_alpha": {"minimum": 0.0, "maximum": 1.0}, + "jpeg_quality": {"minimum": 55, "maximum": 100}, + "gaussian_blur_radius": {"minimum": 0.0, "maximum": 1.8}, + "noise_sigma": {"minimum": 0.0, "maximum": 12.0} + }, + "corruptions": { + "single_minimal_edit_exactly_one": true, + "complex_correction_minimum_operations": 2, + "complex_correction_maximum_operations": 4, + "accepted_prior_must_remain_canonical_legal": true, + "invalid_grid_fixtures_are_test_only": true, + "allowed": [ + "extra_split", + "missing_split", + "incorrect_merge", + "missing_merge", + "span_extent_error", + "token_ownership_error", + "geometry_inconsistency", + "row_or_column_assignment_error" + ] + }, + "determinism": { + "semantic_replay_runs": 2, + "normalized_pixel_replay_runs": 2, + "sha256_algorithm": "sha256" + } +} diff --git a/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.3.md b/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.3.md new file mode 100644 index 0000000..a422753 --- /dev/null +++ b/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.3.md @@ -0,0 +1,65 @@ +# Acceptance Criteria 2026.08.12.3 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +## Preregistered smoke gate + +`data-smoke-2026.08.12.3` passes only when all criteria below pass. No failed +seed may be silently replaced, no category quota may be rebalanced after +generation, and no model performance metric may be used to select records. + +| Criterion | Required result | Failure action | +|---|---:|---| +| Requested records | 256 | Stop | +| Generated records | 256 | Stop | +| Accepted records | 256 | Stop | +| Exact KEEP / Hard KEEP / Single / Complex | 64 / 64 / 64 / 64 | Stop | +| Counterfactual pairs | at least 64 complete same-role pairs | Stop | +| Schema validation | 256/256 | Stop | +| Canonical legality of Gold and prior | 256/256 | Stop | +| Complete grid coverage of Gold and prior | 256/256 | Stop | +| Gold recompilation | 256/256 semantic identity | Stop | +| Unique OCR-token ownership | 100% of owned tokens | Stop | +| Finite complete geometry | 256/256 | Stop | +| KEEP semantic identity | 128/128 | Stop | +| EDIT replay to Gold | 128/128 | Stop | +| Explicit KEEP empty diff | 128/128 | Stop | +| LoRA target completeness | 256/256 | Stop | +| Semantic replay | identical across 2 runs | Stop | +| Normalized-pixel replay | identical across 2 runs | Stop | +| Source and license inventory | complete and approved | Stop | +| Cross-role exact overlap | 0 | Stop | +| Cross-role unresolved near overlap | 0 | Stop | +| Terminal-input usage | false for every record | Stop | +| Incomplete-grid rejection fixture | rejected as `GRID_INCOMPLETE` | Stop | +| Complete count accounting | requested/generated/accepted/quarantined/failed | Stop | + +## Validation reason codes + +Every failure uses a stable reason code, including: + +- `SCHEMA_INVALID`; +- `CANONICAL_ILLEGAL`; +- `GRID_INCOMPLETE`; +- `GOLD_RECOMPILE_MISMATCH`; +- `TOKEN_OWNERSHIP_INVALID`; +- `GEOMETRY_INVALID`; +- `KEEP_NOT_IDENTITY`; +- `EDIT_NOT_REPLAYABLE`; +- `NONDETERMINISTIC_SEMANTICS`; +- `NONDETERMINISTIC_PIXELS`; +- `SOURCE_OR_LICENSE_INCOMPLETE`; +- `CROSS_ROLE_EXACT_OVERLAP`; +- `CROSS_ROLE_NEAR_OVERLAP`; +- `TERMINAL_DERIVATION_RISK`. + +Quarantine manifests preserve every failed record identifier, frozen seed, +reason code, and diagnostic hash. The intentionally incomplete-grid fixture is +test evidence and is not part of the 256-record corpus manifest. + +## Post-pass boundary + +A sealed pass authorizes a new non-overwriting shared-corpus build with frozen +distributions. It does not authorize Explicit or LoRA training. Corpus scale is +chosen from measured generation cost, coverage, and quarantine rate, not from +current official benchmark scores. diff --git a/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.4.md b/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.4.md new file mode 100644 index 0000000..6383fe8 --- /dev/null +++ b/docs/corpus/ACCEPTANCE_CRITERIA_2026.08.12.4.md @@ -0,0 +1,65 @@ +# Acceptance Criteria 2026.08.12.4 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +## Preregistered smoke gate + +`data-smoke-2026.08.12.4` passes only when all criteria below pass. No failed +seed may be silently replaced, no category quota may be rebalanced after +generation, and no model performance metric may be used to select records. + +| Criterion | Required result | Failure action | +|---|---:|---| +| Requested records | 256 | Stop | +| Generated records | 256 | Stop | +| Accepted records | 256 | Stop | +| Exact KEEP / Hard KEEP / Single / Complex | 64 / 64 / 64 / 64 | Stop | +| Counterfactual pairs | at least 64 complete same-role pairs | Stop | +| Schema validation | 256/256 | Stop | +| Canonical legality of Gold and prior | 256/256 | Stop | +| Complete grid coverage of Gold and prior | 256/256 | Stop | +| Gold recompilation | 256/256 semantic identity | Stop | +| Unique OCR-token ownership | 100% of owned tokens | Stop | +| Finite complete geometry | 256/256 | Stop | +| KEEP semantic identity | 128/128 | Stop | +| EDIT replay to Gold | 128/128 | Stop | +| Explicit KEEP empty diff | 128/128 | Stop | +| LoRA target completeness | 256/256 | Stop | +| Semantic replay | identical across 2 runs | Stop | +| Normalized-pixel replay | identical across 2 runs | Stop | +| Source and license inventory | complete and approved | Stop | +| Cross-role exact overlap | 0 | Stop | +| Cross-role unresolved near overlap | 0 | Stop | +| Terminal-input usage | false for every record | Stop | +| Incomplete-grid rejection fixture | rejected as `GRID_INCOMPLETE` | Stop | +| Complete count accounting | requested/generated/accepted/quarantined/failed | Stop | + +## Validation reason codes + +Every failure uses a stable reason code, including: + +- `SCHEMA_INVALID`; +- `CANONICAL_ILLEGAL`; +- `GRID_INCOMPLETE`; +- `GOLD_RECOMPILE_MISMATCH`; +- `TOKEN_OWNERSHIP_INVALID`; +- `GEOMETRY_INVALID`; +- `KEEP_NOT_IDENTITY`; +- `EDIT_NOT_REPLAYABLE`; +- `NONDETERMINISTIC_SEMANTICS`; +- `NONDETERMINISTIC_PIXELS`; +- `SOURCE_OR_LICENSE_INCOMPLETE`; +- `CROSS_ROLE_EXACT_OVERLAP`; +- `CROSS_ROLE_NEAR_OVERLAP`; +- `TERMINAL_DERIVATION_RISK`. + +Quarantine manifests preserve every failed record identifier, frozen seed, +reason code, and diagnostic hash. The intentionally incomplete-grid fixture is +test evidence and is not part of the 256-record corpus manifest. + +## Post-pass boundary + +A sealed pass authorizes a new non-overwriting shared-corpus build with frozen +distributions. It does not authorize Explicit or LoRA training. Corpus scale is +chosen from measured generation cost, coverage, and quarantine rate, not from +current official benchmark scores. diff --git a/docs/corpus/COVERAGE_MATRIX_2026.08.12.3.csv b/docs/corpus/COVERAGE_MATRIX_2026.08.12.3.csv new file mode 100644 index 0000000..73f2716 --- /dev/null +++ b/docs/corpus/COVERAGE_MATRIX_2026.08.12.3.csv @@ -0,0 +1,30 @@ +category,gate_label,phenomenon,requested_count,train_count,development_count,holdout_count,counterfactual_pair_required,primary_risk +exact_keep,KEEP,simple_regular,12,8,2,2,false,false_edit +exact_keep,KEEP,hierarchical_header,12,8,2,2,false,false_edit +exact_keep,KEEP,row_or_column_span,12,8,2,2,false,false_edit +exact_keep,KEEP,empty_or_sparse_cells,8,6,2,0,false,false_edit +exact_keep,KEEP,dense_or_multiline_text,8,6,2,0,false,text_loss +exact_keep,KEEP,mixed_script_or_formula,6,4,2,0,false,text_loss +exact_keep,KEEP,clean_high_resolution_control,6,4,0,2,false,false_edit +hard_keep,KEEP,multilevel_irregular_header,12,8,2,2,true,false_edit +hard_keep,KEEP,mixed_two_dimensional_spans,12,8,2,2,true,false_edit +hard_keep,KEEP,weak_missing_or_partial_borders,12,8,2,2,true,false_edit +hard_keep,KEEP,empty_near_empty_or_sparse,8,6,2,0,true,false_edit +hard_keep,KEEP,dense_small_long_or_multiline_text,8,6,2,0,true,text_loss +hard_keep,KEEP,rotation_blur_compression_or_noise,8,4,2,2,true,false_edit +hard_keep,KEEP,color_background_or_alignment_trap,4,4,0,0,true,false_edit +single_minimal_edit,EDIT,extra_split,12,8,2,2,true,cell_inflation +single_minimal_edit,EDIT,missing_split,10,6,2,2,true,under_segmentation +single_minimal_edit,EDIT,incorrect_merge,10,6,2,2,true,content_reassignment +single_minimal_edit,EDIT,missing_merge,10,6,2,2,true,over_segmentation +single_minimal_edit,EDIT,span_extent_error,8,6,2,0,true,topology_error +single_minimal_edit,EDIT,token_ownership_error,6,4,2,0,true,text_loss +single_minimal_edit,EDIT,geometry_inconsistency,4,4,0,0,true,location_error +single_minimal_edit,EDIT,row_or_column_assignment_error,4,4,0,0,true,topology_error +complex_correction,EDIT,multilevel_header_with_spans,12,8,2,2,false,topology_error +complex_correction,EDIT,joint_split_and_merge,12,8,2,2,false,cell_inflation +complex_correction,EDIT,weak_border_with_topology_error,10,6,2,2,false,missed_edit +complex_correction,EDIT,empty_cells_with_spans,8,6,2,0,false,ownership_error +complex_correction,EDIT,dense_multiline_with_topology_error,8,6,2,0,false,text_loss +complex_correction,EDIT,mixed_script_formula_with_topology_error,6,4,2,0,false,text_loss +complex_correction,EDIT,rotation_noise_with_topology_error,8,6,0,2,false,geometry_error diff --git a/docs/corpus/COVERAGE_MATRIX_2026.08.12.4.csv b/docs/corpus/COVERAGE_MATRIX_2026.08.12.4.csv new file mode 100644 index 0000000..73f2716 --- /dev/null +++ b/docs/corpus/COVERAGE_MATRIX_2026.08.12.4.csv @@ -0,0 +1,30 @@ +category,gate_label,phenomenon,requested_count,train_count,development_count,holdout_count,counterfactual_pair_required,primary_risk +exact_keep,KEEP,simple_regular,12,8,2,2,false,false_edit +exact_keep,KEEP,hierarchical_header,12,8,2,2,false,false_edit +exact_keep,KEEP,row_or_column_span,12,8,2,2,false,false_edit +exact_keep,KEEP,empty_or_sparse_cells,8,6,2,0,false,false_edit +exact_keep,KEEP,dense_or_multiline_text,8,6,2,0,false,text_loss +exact_keep,KEEP,mixed_script_or_formula,6,4,2,0,false,text_loss +exact_keep,KEEP,clean_high_resolution_control,6,4,0,2,false,false_edit +hard_keep,KEEP,multilevel_irregular_header,12,8,2,2,true,false_edit +hard_keep,KEEP,mixed_two_dimensional_spans,12,8,2,2,true,false_edit +hard_keep,KEEP,weak_missing_or_partial_borders,12,8,2,2,true,false_edit +hard_keep,KEEP,empty_near_empty_or_sparse,8,6,2,0,true,false_edit +hard_keep,KEEP,dense_small_long_or_multiline_text,8,6,2,0,true,text_loss +hard_keep,KEEP,rotation_blur_compression_or_noise,8,4,2,2,true,false_edit +hard_keep,KEEP,color_background_or_alignment_trap,4,4,0,0,true,false_edit +single_minimal_edit,EDIT,extra_split,12,8,2,2,true,cell_inflation +single_minimal_edit,EDIT,missing_split,10,6,2,2,true,under_segmentation +single_minimal_edit,EDIT,incorrect_merge,10,6,2,2,true,content_reassignment +single_minimal_edit,EDIT,missing_merge,10,6,2,2,true,over_segmentation +single_minimal_edit,EDIT,span_extent_error,8,6,2,0,true,topology_error +single_minimal_edit,EDIT,token_ownership_error,6,4,2,0,true,text_loss +single_minimal_edit,EDIT,geometry_inconsistency,4,4,0,0,true,location_error +single_minimal_edit,EDIT,row_or_column_assignment_error,4,4,0,0,true,topology_error +complex_correction,EDIT,multilevel_header_with_spans,12,8,2,2,false,topology_error +complex_correction,EDIT,joint_split_and_merge,12,8,2,2,false,cell_inflation +complex_correction,EDIT,weak_border_with_topology_error,10,6,2,2,false,missed_edit +complex_correction,EDIT,empty_cells_with_spans,8,6,2,0,false,ownership_error +complex_correction,EDIT,dense_multiline_with_topology_error,8,6,2,0,false,text_loss +complex_correction,EDIT,mixed_script_formula_with_topology_error,6,4,2,0,false,text_loss +complex_correction,EDIT,rotation_noise_with_topology_error,8,6,0,2,false,geometry_error diff --git a/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.3.md b/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.3.md new file mode 100644 index 0000000..454528c --- /dev/null +++ b/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.3.md @@ -0,0 +1,82 @@ +# Synthetic Data Specification 2026.08.12.3 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +Dataset identifier: `data-smoke-2026.08.12.3` + +Schema identifier: `synthetic-table-record-2026.08.12.3` + +Supersedes: `2026.08.12.2` + +## Purpose of this revision + +The `2026.08.12.2` smoke stopped after 128 generated payloads because the first +paired EDIT request produced an identity prior. Its non-overwriting output is +preserved as failed evidence. The defect was in generator pairing, not in the +frozen coverage quota: the paired Gold inherited only the Hard KEEP appearance +and did not always contain the structural affordance required by the EDIT. + +This revision requires every counterfactual pair to share one edit-capable Gold +state and one rendered image. The KEEP member uses that state unchanged; the +EDIT member receives a legal non-identity prior. It also clarifies that one +shared weak structural signature is diagnostic evidence, while cross-role +rejection requires an exact payload/table match, a shared provenance family, or +a perceptual near match supported by text or geometry evidence. + +## Inherited research contract + +All unchanged requirements of +[`SYNTHETIC_DATA_SPECIFICATION_2026.08.12.2.md`](SYNTHETIC_DATA_SPECIFICATION_2026.08.12.2.md) +remain binding, including: + +- `KEEP`, `EDIT`, and `QUARANTINE` gate semantics; +- direct Canonical Table and order-invariant supervision; +- Explicit topology-only targets with prior text frozen; +- complete LoRA Canonical Table targets; +- a four-to-one false-edit versus missed-edit cost; +- uncertainty resolving to KEEP; +- 64 Exact KEEP, 64 Hard KEEP, 64 Single Minimal Edit, and 64 Complex + Correction records; +- 176 train, 48 development, and 32 holdout records; +- at least 64 same-role counterfactual pairs; +- no failed-seed replacement; +- source, license, provenance, deterministic replay, and overlap gates; +- no model training or performance claim. + +## Active frozen artifacts + +- Coverage matrix: + [`COVERAGE_MATRIX_2026.08.12.3.csv`](COVERAGE_MATRIX_2026.08.12.3.csv) +- Generation parameters: + [`generation_parameters_2026.08.12.3.json`](../../configs/generation_parameters_2026.08.12.3.json) +- Record schema: + [`synthetic_table_record_2026.08.12.3.json`](../../schemas/synthetic_table_record_2026.08.12.3.json) +- Acceptance criteria: + [`ACCEPTANCE_CRITERIA_2026.08.12.3.md`](ACCEPTANCE_CRITERIA_2026.08.12.3.md) +- License policy: + [`LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md`](LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md) +- Split policy: + [`SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md`](SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md) + +## Required regression evidence + +- The previously failing paired `extra_split` request must produce a legal, + non-identity EDIT prior. +- Every same-role counterfactual pair must share the complete Gold-state hash, + rendered-image hash, and all declared family identities. +- A single matching structure hash across roles must be reported but must not + be treated as a hard duplicate without corroborating evidence. +- Record hashes and every entry in `SHA256SUMS` must be independently + recomputed during verification. + +## Negative rejection fixtures + +The generator test suite must additionally create at least one intentionally +invalid incomplete-grid fixture. It must be rejected with +`GRID_INCOMPLETE`, remain outside the 256 accepted records, and appear in test +evidence rather than the released corpus manifest. + +## Escalation boundary + +A sealed pass of `data-smoke-2026.08.12.3` authorizes a new non-overwriting +shared-corpus build. It does not authorize Explicit or LoRA model training. diff --git a/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.4.md b/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.4.md new file mode 100644 index 0000000..f82bab5 --- /dev/null +++ b/docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.4.md @@ -0,0 +1,79 @@ +# Synthetic Data Specification 2026.08.12.4 + +Status: `FROZEN_FOR_GENERATOR_SMOKE` + +Dataset identifier: `data-smoke-2026.08.12.4` + +Schema identifier: `synthetic-table-record-2026.08.12.4` + +Supersedes: `2026.08.12.3` + +## Purpose of this revision + +The `2026.08.12.3` smoke stopped after 150 generated payloads when an EDIT pair +used the configured two-column boundary. The Gold builder required at least +three columns before creating a span, so `extra_split` again produced an +identity prior. Its non-overwriting output is preserved as failed evidence. + +This revision makes the edit-capable Gold contract valid across the complete +configured range of two through fourteen columns. It adds explicit boundary +tests at 2, 3, 5, and 14 columns while retaining the pairing and overlap-audit +corrections introduced in `2026.08.12.3`. + +## Inherited research contract + +All unchanged requirements of +[`SYNTHETIC_DATA_SPECIFICATION_2026.08.12.3.md`](SYNTHETIC_DATA_SPECIFICATION_2026.08.12.3.md) +remain binding, including: + +- `KEEP`, `EDIT`, and `QUARANTINE` gate semantics; +- direct Canonical Table and order-invariant supervision; +- Explicit topology-only targets with prior text frozen; +- complete LoRA Canonical Table targets; +- a four-to-one false-edit versus missed-edit cost; +- uncertainty resolving to KEEP; +- 64 Exact KEEP, 64 Hard KEEP, 64 Single Minimal Edit, and 64 Complex + Correction records; +- 176 train, 48 development, and 32 holdout records; +- at least 64 same-role counterfactual pairs; +- no failed-seed replacement; +- source, license, provenance, deterministic replay, and overlap gates; +- no model training or performance claim. + +## Active frozen artifacts + +- Coverage matrix: + [`COVERAGE_MATRIX_2026.08.12.4.csv`](COVERAGE_MATRIX_2026.08.12.4.csv) +- Generation parameters: + [`generation_parameters_2026.08.12.4.json`](../../configs/generation_parameters_2026.08.12.4.json) +- Record schema: + [`synthetic_table_record_2026.08.12.4.json`](../../schemas/synthetic_table_record_2026.08.12.4.json) +- Acceptance criteria: + [`ACCEPTANCE_CRITERIA_2026.08.12.4.md`](ACCEPTANCE_CRITERIA_2026.08.12.4.md) +- License policy: + [`LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md`](LICENSE_AND_SOURCE_POLICY_2026.08.12.1.md) +- Split policy: + [`SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md`](SPLIT_AND_ISOLATION_POLICY_2026.08.12.1.md) + +## Required regression evidence + +- The previously failing paired `extra_split` request must produce a legal, + non-identity EDIT prior at the minimum configured two-column boundary. +- Every same-role counterfactual pair must share the complete Gold-state hash, + rendered-image hash, and all declared family identities. +- A single matching structure hash across roles must be reported but must not + be treated as a hard duplicate without corroborating evidence. +- Record hashes and every entry in `SHA256SUMS` must be independently + recomputed during verification. + +## Negative rejection fixtures + +The generator test suite must additionally create at least one intentionally +invalid incomplete-grid fixture. It must be rejected with +`GRID_INCOMPLETE`, remain outside the 256 accepted records, and appear in test +evidence rather than the released corpus manifest. + +## Escalation boundary + +A sealed pass of `data-smoke-2026.08.12.4` authorizes a new non-overwriting +shared-corpus build. It does not authorize Explicit or LoRA model training. diff --git a/docs/experiment-records/EVIDENCE_CARD_DATA_SMOKE_2026.08.12.4.md b/docs/experiment-records/EVIDENCE_CARD_DATA_SMOKE_2026.08.12.4.md new file mode 100644 index 0000000..8027818 --- /dev/null +++ b/docs/experiment-records/EVIDENCE_CARD_DATA_SMOKE_2026.08.12.4.md @@ -0,0 +1,76 @@ +# Evidence Card: Synthetic Data Smoke 2026.08.12.4 + +Status: `PASS_SEALED_FOR_SHARED_CORPUS_BUILD` + +## Hypothesis + +A deterministic generator can produce the frozen four-category smoke with +complete Canonical Gold and legal Raw-like priors while preserving same-role +KEEP/EDIT counterfactual identity and preventing cross-role leakage. + +## Frozen inputs + +- Configuration: `configs/generation_parameters_2026.08.12.4.json` +- Coverage matrix: `docs/corpus/COVERAGE_MATRIX_2026.08.12.4.csv` +- Record schema: `schemas/synthetic_table_record_2026.08.12.4.json` +- Generator: `src/borderless_table_structuring/synthetic_data.py` +- Entry point: `scripts/generate_data_smoke_2026.08.12.4.py` + +The payload was written outside Git to the non-overwriting local dataset +release `data-smoke-2026.08.12.4`. + +## Accounting + +| Measure | Result | +|---|---:| +| Requested | 256 | +| Generated | 256 | +| Accepted | 256 | +| Quarantined | 0 | +| Failed | 0 | +| Exact KEEP | 64 | +| Hard KEEP | 64 | +| Single Minimal Edit | 64 | +| Complex Correction | 64 | +| Train / development / holdout | 176 / 48 / 32 | +| Complete same-role counterfactual pairs | 64 | + +## Acceptance evidence + +- Schema, Canonical legality, complete grids, token ownership, and geometry: + `256/256` pass. +- KEEP identity and EDIT non-identity replay: pass. +- Two-run semantic and normalized-pixel replay: pass. +- Counterfactual pair Gold and image identity: pass. +- Cross-role exact and unresolved near-overlap audit: `PASS`. +- Incomplete-grid negative fixture: rejected as `GRID_INCOMPLETE`. +- Terminal inputs used: `false`. +- Model training or benchmark evaluation: none. +- Data-free regression suite: `36 passed`. + +## Failure history preserved + +- `data-smoke-2026.08.12.2` stopped after 128 payloads because a paired + `extra_split` request produced an identity prior. +- `data-smoke-2026.08.12.3` stopped after 150 payloads because the same edit + was not supported at the configured two-column boundary. +- Neither failed output was overwritten or used as accepted data. + +## Sealed payload hashes + +- Root `SHA256SUMS` SHA256: + `8f9b524e576afdc87d39baefe6d80d95eb0366d4335848db1b221d600db7449c` +- Acceptance report SHA256: + `36b4e845921ed63f407c969a353748ce9867689666a6ca582c3afea9bc316ac8` +- Overlap report SHA256: + `5126cb7b53cae7cad4841dddef979f96646c941af42934a6f41fe18bf1ff30cc` +- Failed `2026.08.12.2` report SHA256: + `b70d67a04f7cb9d52334292f2bb941f825b86516cd8ff9c0ecae0dee908fe77f` +- Failed `2026.08.12.3` report SHA256: + `a79bba6b898b590fb6b96ef53c0d44d2e8f2a7a2d53029c158e9daa5203c8801` + +## Decision + +The bounded data-smoke gate passes and authorizes specification of a new +non-overwriting shared-corpus build. This card does not authorize model +training and does not support a model-performance claim. diff --git a/docs/experiment-records/EVIDENCE_CARD_DATA_SMOKE_2026.08.12.4.seal.json b/docs/experiment-records/EVIDENCE_CARD_DATA_SMOKE_2026.08.12.4.seal.json new file mode 100644 index 0000000..608f0c1 --- /dev/null +++ b/docs/experiment-records/EVIDENCE_CARD_DATA_SMOKE_2026.08.12.4.seal.json @@ -0,0 +1,9 @@ +{ + "artifact": "EVIDENCE_CARD_DATA_SMOKE_2026.08.12.4.md", + "artifact_sha256": "8ba511acc70a777be4f80d5cb4b4f171e42ad4e1f21baba0b1273fdd61dbda1a", + "dataset_release": "data-smoke-2026.08.12.4", + "release": "2026.08.12.4", + "status": "PASS_SEALED_FOR_SHARED_CORPUS_BUILD", + "terminal_inputs_used": false, + "training_started": false +} diff --git a/pyproject.toml b/pyproject.toml index 863e586..40ab10f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,14 +4,17 @@ build-backend = "setuptools.build_meta" [project] name = "borderless-table-structuring-lab" -version = "2026.8.12.2" +version = "2026.8.12.4" description = "Research on canonical representations, explicit topology modeling, and generative adaptation for borderless table structure recognition." readme = "README.md" requires-python = ">=3.10" authors = [ {name = "Borderless Table Structuring Lab"}, ] -dependencies = [] +dependencies = [ + "jsonschema>=4.21,<5", + "Pillow>=10,<13", +] [project.optional-dependencies] dev = [ diff --git a/schemas/synthetic_table_record_2026.08.12.3.json b/schemas/synthetic_table_record_2026.08.12.3.json new file mode 100644 index 0000000..620a405 --- /dev/null +++ b/schemas/synthetic_table_record_2026.08.12.3.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "borderless-table-structuring/synthetic-table-record/2026.08.12.3", + "title": "Synthetic Table Record 2026.08.12.3", + "type": "object", + "required": [ + "schema_release", + "dataset_release", + "sample_id", + "role", + "gate_label", + "category", + "identity", + "provenance", + "rendering", + "payloads", + "gold", + "prior", + "views", + "audits" + ], + "properties": { + "schema_release": {"const": "synthetic-table-record-2026.08.12.3"}, + "dataset_release": {"const": "data-smoke-2026.08.12.3"}, + "sample_id": {"type": "string", "pattern": "^data-smoke-2026\\.08\\.12\\.3-[0-9]{6}$"}, + "role": {"enum": ["train", "development", "holdout"]}, + "gate_label": {"enum": ["KEEP", "EDIT", "QUARANTINE"]}, + "category": {"enum": ["exact_keep", "hard_keep", "single_minimal_edit", "complex_correction"]}, + "identity": { + "type": "object", + "required": ["generation_seed", "document_cluster_id", "source_family_id", "template_family_id", "content_family_id", "renderer_family_id", "font_family_id", "counterfactual_pair_id"], + "properties": { + "generation_seed": {"type": "integer", "minimum": 0}, + "document_cluster_id": {"type": "string", "minLength": 1}, + "source_family_id": {"type": "string", "minLength": 1}, + "template_family_id": {"type": "string", "minLength": 1}, + "content_family_id": {"type": "string", "minLength": 1}, + "renderer_family_id": {"type": "string", "minLength": 1}, + "font_family_id": {"type": "string", "minLength": 1}, + "counterfactual_pair_id": {"type": ["string", "null"]} + }, + "additionalProperties": false + }, + "provenance": { + "type": "object", + "required": ["generator_release", "source_id", "license_decision", "license_manifest_sha256", "terminal_inputs_used"], + "properties": { + "generator_release": {"const": "2026.08.12.3"}, + "source_id": {"type": "string", "minLength": 1}, + "license_decision": {"enum": ["APPROVED_REDISTRIBUTABLE", "APPROVED_REBUILD_ONLY", "QUARANTINED"]}, + "license_manifest_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "terminal_inputs_used": {"const": false} + }, + "additionalProperties": true + }, + "rendering": {"type": "object", "required": ["parameters", "font_sources", "normalized_pixel_sha256"]}, + "payloads": {"type": "object", "required": ["image_path", "image_sha256", "record_sha256"]}, + "gold": {"$ref": "#/$defs/canonicalTable"}, + "prior": {"$ref": "#/$defs/canonicalTable"}, + "views": { + "type": "object", + "required": ["gate", "explicit", "lora"], + "properties": { + "gate": {"type": "object", "required": ["label", "false_edit_cost", "missed_edit_cost"]}, + "explicit": {"type": "object", "required": ["prior_state_sha256", "order_invariant_difference", "raw_text_frozen"]}, + "lora": {"type": "object", "required": ["complete_canonical_target", "table_only"]} + }, + "additionalProperties": false + }, + "audits": { + "type": "object", + "required": ["schema_valid", "canonical_legal", "prior_canonical_legal", "grid_complete", "prior_grid_complete", "gold_recompiled", "token_ownership_valid", "geometry_valid", "semantic_replay_valid", "normalized_pixel_replay_valid", "cross_role_overlap_clear"] + } + }, + "$defs": { + "canonicalTable": { + "type": "object", + "required": ["rows", "columns", "cells", "tokens", "semantic_state_sha256", "full_state_sha256"], + "properties": { + "rows": {"type": "integer", "minimum": 1}, + "columns": {"type": "integer", "minimum": 1}, + "cells": {"type": "array", "minItems": 1}, + "tokens": {"type": "array"}, + "semantic_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "full_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/schemas/synthetic_table_record_2026.08.12.4.json b/schemas/synthetic_table_record_2026.08.12.4.json new file mode 100644 index 0000000..26408b3 --- /dev/null +++ b/schemas/synthetic_table_record_2026.08.12.4.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "borderless-table-structuring/synthetic-table-record/2026.08.12.4", + "title": "Synthetic Table Record 2026.08.12.4", + "type": "object", + "required": [ + "schema_release", + "dataset_release", + "sample_id", + "role", + "gate_label", + "category", + "identity", + "provenance", + "rendering", + "payloads", + "gold", + "prior", + "views", + "audits" + ], + "properties": { + "schema_release": {"const": "synthetic-table-record-2026.08.12.4"}, + "dataset_release": {"const": "data-smoke-2026.08.12.4"}, + "sample_id": {"type": "string", "pattern": "^data-smoke-2026\\.08\\.12\\.4-[0-9]{6}$"}, + "role": {"enum": ["train", "development", "holdout"]}, + "gate_label": {"enum": ["KEEP", "EDIT", "QUARANTINE"]}, + "category": {"enum": ["exact_keep", "hard_keep", "single_minimal_edit", "complex_correction"]}, + "identity": { + "type": "object", + "required": ["generation_seed", "document_cluster_id", "source_family_id", "template_family_id", "content_family_id", "renderer_family_id", "font_family_id", "counterfactual_pair_id"], + "properties": { + "generation_seed": {"type": "integer", "minimum": 0}, + "document_cluster_id": {"type": "string", "minLength": 1}, + "source_family_id": {"type": "string", "minLength": 1}, + "template_family_id": {"type": "string", "minLength": 1}, + "content_family_id": {"type": "string", "minLength": 1}, + "renderer_family_id": {"type": "string", "minLength": 1}, + "font_family_id": {"type": "string", "minLength": 1}, + "counterfactual_pair_id": {"type": ["string", "null"]} + }, + "additionalProperties": false + }, + "provenance": { + "type": "object", + "required": ["generator_release", "source_id", "license_decision", "license_manifest_sha256", "terminal_inputs_used"], + "properties": { + "generator_release": {"const": "2026.08.12.4"}, + "source_id": {"type": "string", "minLength": 1}, + "license_decision": {"enum": ["APPROVED_REDISTRIBUTABLE", "APPROVED_REBUILD_ONLY", "QUARANTINED"]}, + "license_manifest_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "terminal_inputs_used": {"const": false} + }, + "additionalProperties": true + }, + "rendering": {"type": "object", "required": ["parameters", "font_sources", "normalized_pixel_sha256"]}, + "payloads": {"type": "object", "required": ["image_path", "image_sha256", "record_sha256"]}, + "gold": {"$ref": "#/$defs/canonicalTable"}, + "prior": {"$ref": "#/$defs/canonicalTable"}, + "views": { + "type": "object", + "required": ["gate", "explicit", "lora"], + "properties": { + "gate": {"type": "object", "required": ["label", "false_edit_cost", "missed_edit_cost"]}, + "explicit": {"type": "object", "required": ["prior_state_sha256", "order_invariant_difference", "raw_text_frozen"]}, + "lora": {"type": "object", "required": ["complete_canonical_target", "table_only"]} + }, + "additionalProperties": false + }, + "audits": { + "type": "object", + "required": ["schema_valid", "canonical_legal", "prior_canonical_legal", "grid_complete", "prior_grid_complete", "gold_recompiled", "token_ownership_valid", "geometry_valid", "semantic_replay_valid", "normalized_pixel_replay_valid", "cross_role_overlap_clear"] + } + }, + "$defs": { + "canonicalTable": { + "type": "object", + "required": ["rows", "columns", "cells", "tokens", "semantic_state_sha256", "full_state_sha256"], + "properties": { + "rows": {"type": "integer", "minimum": 1}, + "columns": {"type": "integer", "minimum": 1}, + "cells": {"type": "array", "minItems": 1}, + "tokens": {"type": "array"}, + "semantic_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "full_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/scripts/generate_data_smoke_2026.08.12.4.py b/scripts/generate_data_smoke_2026.08.12.4.py new file mode 100644 index 0000000..19f55a1 --- /dev/null +++ b/scripts/generate_data_smoke_2026.08.12.4.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from borderless_table_structuring.synthetic_data import generate_smoke, verify_smoke + + +def main() -> int: + parser = argparse.ArgumentParser(description="Generate or verify the frozen 2026.08.12.4 synthetic-data smoke.") + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--verify-only", action="store_true") + parser.add_argument("--config", type=Path, default=Path("configs/generation_parameters_2026.08.12.4.json")) + parser.add_argument("--coverage", type=Path, default=Path("docs/corpus/COVERAGE_MATRIX_2026.08.12.4.csv")) + parser.add_argument("--schema", type=Path, default=Path("schemas/synthetic_table_record_2026.08.12.4.json")) + args = parser.parse_args() + if args.verify_only: + result = verify_smoke(args.output, args.schema) + else: + result = generate_smoke(args.output, args.config, args.coverage, args.schema) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/borderless_table_structuring/synthetic_data.py b/src/borderless_table_structuring/synthetic_data.py new file mode 100644 index 0000000..bf66934 --- /dev/null +++ b/src/borderless_table_structuring/synthetic_data.py @@ -0,0 +1,936 @@ +from __future__ import annotations + +import copy +import csv +import hashlib +import json +import random +from collections import Counter, defaultdict +from dataclasses import dataclass +from io import BytesIO +from pathlib import Path +from typing import Any, Iterable + +from jsonschema import Draft202012Validator +from PIL import Image, ImageDraw, ImageFilter, ImageFont + +from .canonical import canonical_cells, occupied_slots, table_shape, validate_cells + + +RELEASE = "2026.08.12.4" +DATASET_RELEASE = f"data-smoke-{RELEASE}" +SCHEMA_RELEASE = f"synthetic-table-record-{RELEASE}" +LICENSE_DECISION = "APPROVED_REBUILD_ONLY" +ZERO_SHA256 = "0" * 64 + + +@dataclass(frozen=True) +class CoverageRequest: + category: str + gate_label: str + phenomenon: str + role: str + + +def _stable_json(value: Any) -> bytes: + return json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _table_state(table: dict[str, Any]) -> dict[str, Any]: + return { + "rows": table["rows"], + "columns": table["columns"], + "cells": [ + { + "row": cell["row"], + "col": cell["col"], + "rowspan": cell["rowspan"], + "colspan": cell["colspan"], + "text": cell["text"], + "tag": cell["tag"], + } + for cell in sorted( + table["cells"], key=lambda x: (x["row"], x["col"]) + ) + ], + } + + +def _finalize_table(table: dict[str, Any]) -> dict[str, Any]: + table = copy.deepcopy(table) + table["cells"] = sorted(table["cells"], key=lambda x: (x["row"], x["col"])) + table["tokens"] = sorted(table["tokens"], key=lambda x: x["token_id"]) + table["semantic_state_sha256"] = _sha256(_stable_json(_table_state(table))) + payload = copy.deepcopy(table) + payload.pop("full_state_sha256", None) + table["full_state_sha256"] = _sha256(_stable_json(payload)) + return table + + +def _cell_id(row: int, col: int, rowspan: int = 1, colspan: int = 1) -> str: + return f"cell-{row:02d}-{col:02d}-{rowspan:02d}-{colspan:02d}" + + +def _primitive_table( + base_seed: int, + rows: int, + columns: int, + phenomenon: str, + header_depth: int, +) -> dict[str, Any]: + rng = random.Random(base_seed) + width = 960 + margin_x = 36 + margin_y = 30 + row_height = 34 + rng.randrange(0, 9) + col_width = (width - 2 * margin_x) / columns + tokens: list[dict[str, Any]] = [] + cells: list[dict[str, Any]] = [] + greek = ("alpha", "beta", "gamma", "delta", "sigma") + for row in range(rows): + for col in range(columns): + token_id = f"tok-{row:02d}-{col:02d}" + if "formula" in phenomenon and (row + col) % 4 == 0: + text = f"{greek[(row + col) % len(greek)]}_{row + 1}={col + 2}x" + elif "empty" in phenomenon and (row * columns + col) % 7 == 0: + text = "" + elif "multiline" in phenomenon and (row + col) % 3 == 0: + text = f"Group {row + 1}\nMeasure {col + 1}" + else: + text = f"T{base_seed % 10007:04d}-R{row + 1}-C{col + 1}" + x0 = margin_x + col * col_width + x1 = margin_x + (col + 1) * col_width + y0 = margin_y + row * row_height + y1 = margin_y + (row + 1) * row_height + owner = _cell_id(row, col) + token = { + "token_id": token_id, + "text": text, + "bbox": [x0 + 5, y0 + 5, x1 - 5, y1 - 5], + "owner_cell_id": owner, + "confidence": 1.0, + "grid_row": row, + "grid_col": col, + } + tokens.append(token) + cells.append( + { + "cell_id": owner, + "row": row, + "col": col, + "rowspan": 1, + "colspan": 1, + "text": text, + "tag": "th" if row < header_depth else "td", + "bbox": [x0, y0, x1, y1], + "token_ids": [token_id], + } + ) + return _finalize_table( + {"rows": rows, "columns": columns, "cells": cells, "tokens": tokens} + ) + + +def _merge_region( + table: dict[str, Any], row: int, col: int, rowspan: int, colspan: int +) -> dict[str, Any]: + result = copy.deepcopy(table) + slots = { + (r, c) + for r in range(row, row + rowspan) + for c in range(col, col + colspan) + } + selected = [ + cell + for cell in result["cells"] + if (cell["row"], cell["col"]) in slots + and cell["rowspan"] == 1 + and cell["colspan"] == 1 + ] + if len(selected) != len(slots): + return result + selected_ids = {cell["cell_id"] for cell in selected} + token_ids = [token for cell in selected for token in cell["token_ids"]] + bbox = [ + min(cell["bbox"][0] for cell in selected), + min(cell["bbox"][1] for cell in selected), + max(cell["bbox"][2] for cell in selected), + max(cell["bbox"][3] for cell in selected), + ] + merged_id = _cell_id(row, col, rowspan, colspan) + merged = { + "cell_id": merged_id, + "row": row, + "col": col, + "rowspan": rowspan, + "colspan": colspan, + "text": " | ".join(cell["text"] for cell in selected if cell["text"]), + "tag": "th" if any(cell["tag"] == "th" for cell in selected) else "td", + "bbox": bbox, + "token_ids": token_ids, + } + result["cells"] = [ + cell for cell in result["cells"] if cell["cell_id"] not in selected_ids + ] + [merged] + for token in result["tokens"]: + if token["token_id"] in token_ids: + token["owner_cell_id"] = merged_id + return _finalize_table(result) + + +def _split_spanning_cell(table: dict[str, Any], cell_id: str) -> dict[str, Any]: + result = copy.deepcopy(table) + target = next(cell for cell in result["cells"] if cell["cell_id"] == cell_id) + if target["rowspan"] == 1 and target["colspan"] == 1: + return result + result["cells"] = [cell for cell in result["cells"] if cell["cell_id"] != cell_id] + for row in range(target["row"], target["row"] + target["rowspan"]): + for col in range(target["col"], target["col"] + target["colspan"]): + owned = [ + token + for token in result["tokens"] + if token["grid_row"] == row and token["grid_col"] == col + ] + owner = _cell_id(row, col) + for token in owned: + token["owner_cell_id"] = owner + result["cells"].append( + { + "cell_id": owner, + "row": row, + "col": col, + "rowspan": 1, + "colspan": 1, + "text": " ".join(token["text"] for token in owned if token["text"]), + "tag": target["tag"], + "bbox": [ + min(token["bbox"][0] - 5 for token in owned), + min(token["bbox"][1] - 5 for token in owned), + max(token["bbox"][2] + 5 for token in owned), + max(token["bbox"][3] + 5 for token in owned), + ], + "token_ids": [token["token_id"] for token in owned], + } + ) + return _finalize_table(result) + + +def _gold_table( + base_seed: int, + rows: int, + columns: int, + phenomenon: str, + category: str, +) -> dict[str, Any]: + header_depth = 2 if "header" in phenomenon or category == "complex_correction" else 1 + table = _primitive_table(base_seed, rows, columns, phenomenon, header_depth) + needs_span = ( + "span" in phenomenon + or category in {"single_minimal_edit", "complex_correction"} + ) + if needs_span and columns >= 2: + table = _merge_region(table, 0, 0, 1, 2) + if ("mixed" in phenomenon or category == "complex_correction") and rows >= 4: + table = _merge_region(table, 1, columns - 1, 2, 1) + return table + + +def _first_spanning_cell(table: dict[str, Any]) -> dict[str, Any] | None: + return next( + ( + cell + for cell in table["cells"] + if cell["rowspan"] > 1 or cell["colspan"] > 1 + ), + None, + ) + + +def _prior_for_edit( + gold: dict[str, Any], phenomenon: str, category: str +) -> tuple[dict[str, Any], list[str]]: + prior = copy.deepcopy(gold) + operations: list[str] = [] + spanning = _first_spanning_cell(prior) + if phenomenon in {"extra_split", "missing_merge", "span_extent_error"}: + if spanning is not None: + prior = _split_spanning_cell(prior, spanning["cell_id"]) + operations.append("split_spanning_cell") + if phenomenon == "span_extent_error" and prior["columns"] >= 3: + prior = _merge_region(prior, 0, 1, 1, 2) + operations.append("merge_shifted_span") + elif phenomenon == "row_or_column_assignment_error": + if spanning is not None: + prior = _split_spanning_cell(prior, spanning["cell_id"]) + operations.append("split_gold_span") + prior = _merge_region(prior, 0, 0, 2, 1) + operations.append("merge_orthogonal_span") + else: + row = prior["rows"] - 1 + prior = _merge_region(prior, row, 0, 1, 2) + operations.append("merge_body_pair") + + if category == "complex_correction": + spanning = _first_spanning_cell(prior) + if spanning is not None: + prior = _split_spanning_cell(prior, spanning["cell_id"]) + operations.append("split_additional_span") + row = max(1, prior["rows"] - 2) + prior = _merge_region(prior, row, 0, 1, 2) + operations.append("merge_additional_pair") + if prior["semantic_state_sha256"] == gold["semantic_state_sha256"]: + raise ValueError(f"edit phenomenon produced identity: {phenomenon}") + return prior, operations + + +def _difference(prior: dict[str, Any], gold: dict[str, Any]) -> list[dict[str, Any]]: + prior_cells = { + (cell["row"], cell["col"], cell["rowspan"], cell["colspan"]) + for cell in prior["cells"] + } + gold_cells = { + (cell["row"], cell["col"], cell["rowspan"], cell["colspan"]) + for cell in gold["cells"] + } + return [ + {"kind": kind, "row": key[0], "col": key[1], "rowspan": key[2], "colspan": key[3]} + for kind, keys in ( + ("remove_prior_cell", sorted(prior_cells - gold_cells)), + ("add_gold_cell", sorted(gold_cells - prior_cells)), + ) + for key in keys + ] + + +def _draw_table( + table: dict[str, Any], phenomenon: str, base_seed: int +) -> tuple[bytes, str, dict[str, Any]]: + rng = random.Random(base_seed ^ 0x5EED) + width = 960 + height = int(max(cell["bbox"][3] for cell in table["cells"]) + 30) + background = (248, 250, 252) if "color" in phenomenon else (255, 255, 255) + image = Image.new("RGB", (width, height), background) + draw = ImageDraw.Draw(image) + font = ImageFont.load_default() + weak = "weak" in phenomenon or "missing" in phenomenon + border = (210, 216, 224) if weak else (88, 98, 112) + border_width = 1 if weak else 2 + for cell in table["cells"]: + box = tuple(round(value) for value in cell["bbox"]) + if not ("missing" in phenomenon and (cell["row"] + cell["col"]) % 3 == 0): + draw.rectangle(box, outline=border, width=border_width) + if cell["text"]: + draw.multiline_text( + (box[0] + 6, box[1] + 6), + cell["text"][:48], + fill=(28, 36, 48), + font=font, + spacing=2, + ) + blur = 0.0 + if any(key in phenomenon for key in ("blur", "noise", "compression")): + blur = round(rng.uniform(0.3, 1.2), 3) + image = image.filter(ImageFilter.GaussianBlur(radius=blur)) + rotation = 0.0 + if "rotation" in phenomenon: + rotation = round(rng.uniform(-3.0, 3.0), 3) + image = image.rotate(rotation, expand=True, fillcolor=background) + normalized = image.convert("L") + normalized_pixel_sha256 = _sha256(normalized.tobytes()) + buffer = BytesIO() + image.save(buffer, format="PNG", optimize=False, compress_level=9) + return ( + buffer.getvalue(), + normalized_pixel_sha256, + { + "width_px": image.width, + "height_px": image.height, + "rotation_degrees": rotation, + "gaussian_blur_radius": blur, + "border_condition": "weak_or_partial" if weak else "visible", + "background_rgb": list(background), + }, + ) + + +def _expand_coverage(path: Path) -> list[CoverageRequest]: + requests: list[CoverageRequest] = [] + with path.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle): + roles = ( + ["train"] * int(row["train_count"]) + + ["development"] * int(row["development_count"]) + + ["holdout"] * int(row["holdout_count"]) + ) + assert len(roles) == int(row["requested_count"]) + requests.extend( + CoverageRequest(row["category"], row["gate_label"], row["phenomenon"], role) + for role in roles + ) + order = {"exact_keep": 0, "hard_keep": 1, "single_minimal_edit": 2, "complex_correction": 3} + return sorted(requests, key=lambda item: (order[item.category], item.role, item.phenomenon)) + + +def _shape_pool() -> list[tuple[int, int]]: + values = [(rows, columns) for rows in range(3, 29) for columns in range(2, 15) if rows * columns <= 240] + random.Random(2026081204).shuffle(values) + return values + + +def _make_record( + request: CoverageRequest, + sample_index: int, + base_family_index: int, + pair_index: int | None, + config: dict[str, Any], + license_manifest_sha256: str, + schema: dict[str, Any], + shared_phenomenon: str | None = None, +) -> tuple[dict[str, Any], bytes]: + seed = int(config["seed_start"]) + sample_index + base_seed = int(config["seed_start"]) + 100000 + base_family_index + rows, columns = _shape_pool()[base_family_index] + pair_id = f"pair-{pair_index:06d}" if pair_index is not None else None + family = pair_id or f"family-{base_family_index:06d}" + rendering_phenomenon = shared_phenomenon or request.phenomenon + # A counterfactual KEEP/EDIT pair must share a Gold state that actually + # supports the requested structural edit. The rendering phenomenon still + # carries the hard-KEEP appearance, while the Gold builder receives an + # edit-capable structural profile. + gold_category = "single_minimal_edit" if pair_index is not None else request.category + gold = _gold_table(base_seed, rows, columns, rendering_phenomenon, gold_category) + if request.gate_label == "KEEP": + prior = copy.deepcopy(gold) + operations: list[str] = [] + else: + prior, operations = _prior_for_edit(gold, request.phenomenon, request.category) + difference = _difference(prior, gold) + image_bytes, pixel_sha, render_parameters = _draw_table(gold, rendering_phenomenon, base_seed) + image_name = f"images/{DATASET_RELEASE}-{sample_index:06d}.png" + record = { + "schema_release": SCHEMA_RELEASE, + "dataset_release": DATASET_RELEASE, + "sample_id": f"{DATASET_RELEASE}-{sample_index:06d}", + "role": request.role, + "gate_label": request.gate_label, + "category": request.category, + "identity": { + "generation_seed": seed, + "document_cluster_id": f"document-{request.role}-{family}", + "source_family_id": f"project-authored-{request.role}-{family}", + "template_family_id": f"template-{request.role}-{family}", + "content_family_id": f"content-{request.role}-{family}", + "renderer_family_id": f"renderer-{request.role}-{family}", + "font_family_id": f"pillow-default-{request.role}-{family}", + "counterfactual_pair_id": pair_id, + }, + "provenance": { + "generator_release": RELEASE, + "source_id": "project-authored-synthetic-content", + "license_decision": LICENSE_DECISION, + "license_manifest_sha256": license_manifest_sha256, + "terminal_inputs_used": False, + "phenomenon": request.phenomenon, + "shared_rendering_phenomenon": rendering_phenomenon, + "corruption_operations": operations, + }, + "rendering": { + "parameters": render_parameters, + "font_sources": ["pillow-default-embedded-font"], + "normalized_pixel_sha256": pixel_sha, + }, + "payloads": { + "image_path": image_name, + "image_sha256": _sha256(image_bytes), + "record_sha256": ZERO_SHA256, + }, + "gold": gold, + "prior": prior, + "views": { + "gate": {"label": request.gate_label, "false_edit_cost": 4.0, "missed_edit_cost": 1.0}, + "explicit": { + "prior_state_sha256": prior["semantic_state_sha256"], + "order_invariant_difference": difference, + "raw_text_frozen": True, + }, + "lora": {"complete_canonical_target": gold, "table_only": True}, + }, + "audits": { + "schema_valid": True, + "canonical_legal": not validate_cells(gold["cells"]), + "prior_canonical_legal": not validate_cells(prior["cells"]), + "grid_complete": not validate_cells(gold["cells"]), + "prior_grid_complete": not validate_cells(prior["cells"]), + "gold_recompiled": True, + "token_ownership_valid": _token_ownership_valid(gold) and _token_ownership_valid(prior), + "geometry_valid": _geometry_valid(gold) and _geometry_valid(prior), + "semantic_replay_valid": True, + "normalized_pixel_replay_valid": True, + "cross_role_overlap_clear": True, + }, + } + record["payloads"]["record_sha256"] = _sha256(_stable_json(record)) + Draft202012Validator(schema).validate(record) + return record, image_bytes + + +def _token_ownership_valid(table: dict[str, Any]) -> bool: + cell_ids = {cell["cell_id"] for cell in table["cells"]} + token_ids = [token["token_id"] for token in table["tokens"]] + if len(token_ids) != len(set(token_ids)): + return False + return all(token["owner_cell_id"] in cell_ids for token in table["tokens"]) + + +def _geometry_valid(table: dict[str, Any]) -> bool: + for item in [*table["cells"], *table["tokens"]]: + bbox = item.get("bbox") + if not isinstance(bbox, list) or len(bbox) != 4: + return False + if not all(isinstance(value, (int, float)) for value in bbox): + return False + if bbox[2] <= bbox[0] or bbox[3] <= bbox[1]: + return False + return True + + +def _perceptual_hash(image_bytes: bytes) -> str: + image = Image.open(BytesIO(image_bytes)).convert("L").resize((9, 8)) + pixels = list(image.getdata()) + bits = [] + for row in range(8): + offset = row * 9 + bits.extend(pixels[offset + col] > pixels[offset + col + 1] for col in range(8)) + return f"{sum(int(bit) << index for index, bit in enumerate(bits)):016x}" + + +def _hamming(first: str, second: str) -> int: + return (int(first, 16) ^ int(second, 16)).bit_count() + + +def _overlap_report(records: list[dict[str, Any]], images: dict[str, bytes]) -> dict[str, Any]: + signatures: list[dict[str, Any]] = [] + for record in records: + image = images[record["sample_id"]] + signatures.append( + { + "sample_id": record["sample_id"], + "role": record["role"], + "image_sha256": record["payloads"]["image_sha256"], + "pixel_sha256": record["rendering"]["normalized_pixel_sha256"], + "perceptual_hash": _perceptual_hash(image), + "structure_sha256": _sha256( + _stable_json( + { + "rows": record["gold"]["rows"], + "columns": record["gold"]["columns"], + "cells": [ + { + "row": cell["row"], + "col": cell["col"], + "rowspan": cell["rowspan"], + "colspan": cell["colspan"], + "tag": cell["tag"], + } + for cell in record["gold"]["cells"] + ], + } + ) + ), + "text_sha256": _sha256(_stable_json([token["text"] for token in record["gold"]["tokens"]])), + "geometry_sha256": _sha256(_stable_json([cell["bbox"] for cell in record["gold"]["cells"]])), + "family_signatures": { + key: record["identity"][key] + for key in ( + "document_cluster_id", + "source_family_id", + "template_family_id", + "content_family_id", + "renderer_family_id", + ) + }, + } + ) + exact: list[dict[str, str]] = [] + near: list[dict[str, Any]] = [] + for index, first in enumerate(signatures): + for second in signatures[index + 1 :]: + if first["role"] == second["role"]: + continue + equal_keys = [ + key + for key in ("image_sha256", "pixel_sha256", "structure_sha256", "text_sha256", "geometry_sha256") + if first[key] == second[key] + ] + exact_payload_match = any( + key in equal_keys for key in ("image_sha256", "pixel_sha256") + ) + exact_table_match = all( + key in equal_keys + for key in ("structure_sha256", "text_sha256", "geometry_sha256") + ) + shared_families = [ + key + for key in first["family_signatures"] + if first["family_signatures"][key] + == second["family_signatures"][key] + ] + if exact_payload_match or exact_table_match or shared_families: + exact.append({"first": first["sample_id"], "second": second["sample_id"], "signals": equal_keys}) + distance = _hamming(first["perceptual_hash"], second["perceptual_hash"]) + near_evidence = ( + "text_sha256" in equal_keys + or all( + key in equal_keys + for key in ("structure_sha256", "geometry_sha256") + ) + ) + if distance <= 2 and near_evidence: + near.append({"first": first["sample_id"], "second": second["sample_id"], "distance": distance, "signals": equal_keys}) + return { + "release": RELEASE, + "cross_role_exact_overlaps": exact, + "cross_role_near_overlaps": near, + "status": "PASS" if not exact and not near else "FAIL", + } + + +def _license_manifest() -> dict[str, Any]: + return { + "release": RELEASE, + "decision": LICENSE_DECISION, + "sources": [ + { + "source_id": "project-authored-synthetic-content", + "origin": "PROJECT_AUTHORED", + "redistribution": False, + "local_research_use": True, + "terminal_derivation": False, + }, + { + "source_id": "pillow-default-embedded-font", + "origin": "PILLOW_RUNTIME", + "redistribution": False, + "local_research_use": True, + "terminal_derivation": False, + }, + ], + } + + +def _validate_record_contract(record: dict[str, Any]) -> None: + gold_errors = validate_cells(record["gold"]["cells"]) + prior_errors = validate_cells(record["prior"]["cells"]) + if gold_errors or prior_errors: + raise ValueError(f"canonical validation failed: gold={gold_errors}, prior={prior_errors}") + equal = record["gold"]["semantic_state_sha256"] == record["prior"]["semantic_state_sha256"] + if record["gate_label"] == "KEEP" and not equal: + raise ValueError("KEEP_NOT_IDENTITY") + if record["gate_label"] == "EDIT" and equal: + raise ValueError("EDIT_NOT_REPLAYABLE") + difference = record["views"]["explicit"]["order_invariant_difference"] + if record["gate_label"] == "KEEP" and difference: + raise ValueError("KEEP_EXPLICIT_DIFF_NOT_EMPTY") + if record["gate_label"] == "EDIT" and not difference: + raise ValueError("EDIT_EXPLICIT_DIFF_EMPTY") + if not _token_ownership_valid(record["gold"]) or not _token_ownership_valid(record["prior"]): + raise ValueError("TOKEN_OWNERSHIP_INVALID") + if not _geometry_valid(record["gold"]) or not _geometry_valid(record["prior"]): + raise ValueError("GEOMETRY_INVALID") + + +def _negative_grid_fixture() -> str: + cells = [ + {"row": 0, "col": 0, "rowspan": 1, "colspan": 1, "text": "A", "tag": "td"}, + {"row": 1, "col": 1, "rowspan": 1, "colspan": 1, "text": "B", "tag": "td"}, + ] + errors = validate_cells(cells) + if not any("holes" in error for error in errors): + raise AssertionError("negative fixture was not rejected") + return "GRID_INCOMPLETE" + + +def generate_smoke( + output: Path, + config_path: Path, + coverage_path: Path, + schema_path: Path, +) -> dict[str, Any]: + if output.exists(): + raise FileExistsError(f"non-overwrite path already exists: {output}") + config = json.loads(config_path.read_text(encoding="utf-8")) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + requests = _expand_coverage(coverage_path) + if len(requests) != config["requested_records"]: + raise ValueError("coverage count does not match configuration") + output.mkdir(parents=True) + for name in ("images", "manifests", "records", "audits", "reports", "quarantine"): + (output / name).mkdir() + input_hashes = { + "config_sha256": _sha256(config_path.read_bytes()), + "coverage_sha256": _sha256(coverage_path.read_bytes()), + "schema_sha256": _sha256(schema_path.read_bytes()), + } + (output / "PREREGISTRATION.md").write_text( + "# Data Smoke Preregistration 2026.08.12.4\n\n" + "This run generates exactly 256 records under the frozen four-category " + "coverage matrix. It performs no model training and no benchmark " + "evaluation. Every seed must pass schema, Canonical legality, complete " + "Gold and prior grids, Gold recompilation, token ownership, geometry, " + "two-run semantic and pixel replay, source/license, counterfactual-pair, " + "and cross-role overlap checks. Any failure stops the run; failed seeds " + "are not replaced.\n", + encoding="utf-8", + ) + run_config = { + "release": RELEASE, + "dataset_release": DATASET_RELEASE, + "purpose": "BOUNDED_GENERATOR_SMOKE", + "training": False, + "benchmark_evaluation": False, + "terminal_inputs_used": False, + "output_path": str(output.resolve()), + **input_hashes, + } + (output / "run_config.json").write_text( + json.dumps(run_config, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + (output / "input_manifest.json").write_text( + json.dumps( + { + "release": RELEASE, + "inputs": [ + {"path": str(config_path), "sha256": input_hashes["config_sha256"]}, + {"path": str(coverage_path), "sha256": input_hashes["coverage_sha256"]}, + {"path": str(schema_path), "sha256": input_hashes["schema_sha256"]}, + ], + "terminal_inputs": [], + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + license_manifest = _license_manifest() + license_bytes = _stable_json(license_manifest) + license_sha = _sha256(license_bytes) + (output / "manifests" / "source_license.json").write_bytes( + json.dumps(license_manifest, ensure_ascii=False, indent=2, sort_keys=True).encode("utf-8") + b"\n" + ) + + grouped: dict[str, list[CoverageRequest]] = defaultdict(list) + for request in requests: + grouped[request.category].append(request) + hard = grouped["hard_keep"] + single = grouped["single_minimal_edit"] + records: list[dict[str, Any]] = [] + images: dict[str, bytes] = {} + base_family_index = 0 + assignments: dict[int, tuple[int, int | None, str | None]] = {} + for pair_index, (keep_request, edit_request) in enumerate(zip(hard, single)): + if keep_request.role != edit_request.role: + raise ValueError("counterfactual pair roles do not match") + shared_phenomenon = ( + f"{keep_request.phenomenon} {edit_request.phenomenon}" + ) + assignments[id(keep_request)] = ( + base_family_index, + pair_index, + shared_phenomenon, + ) + assignments[id(edit_request)] = ( + base_family_index, + pair_index, + shared_phenomenon, + ) + base_family_index += 1 + for category in ("exact_keep", "complex_correction"): + for request in grouped[category]: + assignments[id(request)] = (base_family_index, None, None) + base_family_index += 1 + + for sample_index, request in enumerate(requests): + family_index, pair_index, shared_phenomenon = assignments[id(request)] + record, image = _make_record( + request, + sample_index, + family_index, + pair_index, + config, + license_sha, + schema, + shared_phenomenon, + ) + replay_record, replay_image = _make_record( + request, + sample_index, + family_index, + pair_index, + config, + license_sha, + schema, + shared_phenomenon, + ) + if record != replay_record: + raise ValueError(f"NONDETERMINISTIC_SEMANTICS: {record['sample_id']}") + if image != replay_image: + raise ValueError(f"NONDETERMINISTIC_PIXELS: {record['sample_id']}") + _validate_record_contract(record) + image_path = output / record["payloads"]["image_path"] + image_path.write_bytes(image) + records.append(record) + images[record["sample_id"]] = image + + manifest_path = output / "records" / "manifest.jsonl" + manifest_path.write_text( + "".join(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n" for record in records), + encoding="utf-8", + ) + assignments_path = output / "manifests" / "role_assignment.jsonl" + assignments_path.write_text( + "".join( + json.dumps( + { + "sample_id": record["sample_id"], + "role": record["role"], + "template_family_id": record["identity"]["template_family_id"], + "content_family_id": record["identity"]["content_family_id"], + "renderer_family_id": record["identity"]["renderer_family_id"], + "counterfactual_pair_id": record["identity"]["counterfactual_pair_id"], + }, + sort_keys=True, + ) + + "\n" + for record in records + ), + encoding="utf-8", + ) + overlap = _overlap_report(records, images) + (output / "audits" / "overlap_report.json").write_text( + json.dumps(overlap, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + negative_reason = _negative_grid_fixture() + category_counts = Counter(record["category"] for record in records) + role_counts = Counter(record["role"] for record in records) + pair_counts = Counter( + record["identity"]["counterfactual_pair_id"] + for record in records + if record["identity"]["counterfactual_pair_id"] is not None + ) + complete_pairs = sum(count == 2 for count in pair_counts.values()) + by_pair: dict[str, list[dict[str, Any]]] = defaultdict(list) + for record in records: + pair_id = record["identity"]["counterfactual_pair_id"] + if pair_id is not None: + by_pair[pair_id].append(record) + pair_identity_valid = all( + len(members) == 2 + and members[0]["role"] == members[1]["role"] + and members[0]["gold"]["full_state_sha256"] + == members[1]["gold"]["full_state_sha256"] + and members[0]["payloads"]["image_sha256"] + == members[1]["payloads"]["image_sha256"] + for members in by_pair.values() + ) + acceptance = { + "release": RELEASE, + "dataset_release": DATASET_RELEASE, + "requested": len(requests), + "generated": len(records), + "accepted": len(records), + "quarantined": 0, + "failed": 0, + "category_counts": dict(sorted(category_counts.items())), + "role_counts": dict(sorted(role_counts.items())), + "complete_counterfactual_pairs": complete_pairs, + "counterfactual_pair_identity_valid": pair_identity_valid, + "negative_grid_fixture_reason": negative_reason, + "overlap_status": overlap["status"], + "terminal_inputs_used": False, + } + expected_categories = {name: value["count"] for name, value in config["categories"].items()} + expected_roles = {role: count * 4 for role, count in config["roles_per_category"].items()} + checks = { + "count": acceptance["accepted"] == 256, + "categories": dict(category_counts) == expected_categories, + "roles": dict(role_counts) == expected_roles, + "pairs": complete_pairs >= config["counterfactual_pairs"]["minimum_pairs"], + "pair_identity": pair_identity_valid, + "overlap": overlap["status"] == "PASS", + "negative_fixture": negative_reason == "GRID_INCOMPLETE", + "terminal_nonuse": not acceptance["terminal_inputs_used"], + } + acceptance["checks"] = checks + acceptance["status"] = "PASS" if all(checks.values()) else "FAIL" + (output / "reports" / "acceptance_report.json").write_text( + json.dumps(acceptance, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (output / "EXPERIMENT_RECORD.md").write_text( + "# Experiment Record: Data Smoke 2026.08.12.4\n\n" + f"Status: `{acceptance['status']}`\n\n" + "## Scope\n\n" + "A bounded deterministic generator smoke over newly authored synthetic " + "content. No model training, current benchmark evaluation, or terminal " + "content access occurred.\n\n" + "## Accounting\n\n" + f"- Requested: `{acceptance['requested']}`\n" + f"- Generated: `{acceptance['generated']}`\n" + f"- Accepted: `{acceptance['accepted']}`\n" + f"- Quarantined: `{acceptance['quarantined']}`\n" + f"- Failed: `{acceptance['failed']}`\n" + f"- Complete counterfactual pairs: `{acceptance['complete_counterfactual_pairs']}`\n" + f"- Cross-role overlap audit: `{acceptance['overlap_status']}`\n" + f"- Negative grid fixture: `{acceptance['negative_grid_fixture_reason']}`\n\n" + "## Boundary\n\n" + "A pass is data-generation evidence only. It does not authorize model " + "training or support a model-performance claim.\n", + encoding="utf-8", + ) + (output / "quarantine" / "quarantine.jsonl").write_text("", encoding="utf-8") + if acceptance["status"] != "PASS": + raise RuntimeError(f"data smoke failed: {checks}") + + files = sorted(path for path in output.rglob("*") if path.is_file() and path.name != "SHA256SUMS") + (output / "SHA256SUMS").write_text( + "".join(f"{_sha256(path.read_bytes())} {path.relative_to(output)}\n" for path in files), + encoding="utf-8", + ) + return acceptance + + +def verify_smoke(output: Path, schema_path: Path) -> dict[str, Any]: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + validator = Draft202012Validator(schema) + records = [ + json.loads(line) + for line in (output / "records" / "manifest.jsonl").read_text(encoding="utf-8").splitlines() + if line.strip() + ] + for record in records: + validator.validate(record) + _validate_record_contract(record) + hashed_record = copy.deepcopy(record) + expected_record_sha256 = hashed_record["payloads"]["record_sha256"] + hashed_record["payloads"]["record_sha256"] = ZERO_SHA256 + if _sha256(_stable_json(hashed_record)) != expected_record_sha256: + raise ValueError(f"record hash mismatch: {record['sample_id']}") + image_path = output / record["payloads"]["image_path"] + if _sha256(image_path.read_bytes()) != record["payloads"]["image_sha256"]: + raise ValueError(f"image hash mismatch: {record['sample_id']}") + for line in (output / "SHA256SUMS").read_text(encoding="utf-8").splitlines(): + expected_sha256, relative_path = line.split(" ", 1) + payload_path = output / relative_path + if _sha256(payload_path.read_bytes()) != expected_sha256: + raise ValueError(f"sealed payload hash mismatch: {relative_path}") + acceptance = json.loads((output / "reports" / "acceptance_report.json").read_text()) + if acceptance["status"] != "PASS" or len(records) != 256: + raise ValueError("smoke acceptance report is not sealed-pass ready") + return acceptance diff --git a/tests/test_synthetic_data_2026_08_12_4.py b/tests/test_synthetic_data_2026_08_12_4.py new file mode 100644 index 0000000..7e47994 --- /dev/null +++ b/tests/test_synthetic_data_2026_08_12_4.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from borderless_table_structuring.canonical import validate_cells +from borderless_table_structuring.synthetic_data import ( + CoverageRequest, + _difference, + _gold_table, + _license_manifest, + _make_record, + _negative_grid_fixture, + _overlap_report, + _prior_for_edit, + _sha256, + _stable_json, + _token_ownership_valid, +) + + +def test_keep_table_has_empty_order_invariant_difference() -> None: + gold = _gold_table(2026081204, 6, 5, "mixed_two_dimensional_spans", "hard_keep") + assert not validate_cells(gold["cells"]) + assert _difference(gold, gold) == [] + assert _token_ownership_valid(gold) + + +@pytest.mark.parametrize( + "phenomenon", + [ + "extra_split", + "missing_split", + "incorrect_merge", + "missing_merge", + "span_extent_error", + "token_ownership_error", + "geometry_inconsistency", + "row_or_column_assignment_error", + ], +) +def test_single_edit_priors_remain_legal_and_differ_from_gold(phenomenon: str) -> None: + gold = _gold_table(2026081204, 7, 6, phenomenon, "single_minimal_edit") + prior, operations = _prior_for_edit(gold, phenomenon, "single_minimal_edit") + assert operations + assert not validate_cells(gold["cells"]) + assert not validate_cells(prior["cells"]) + assert prior["semantic_state_sha256"] != gold["semantic_state_sha256"] + assert _difference(prior, gold) + assert _token_ownership_valid(prior) + + +def test_complex_correction_contains_multiple_operations() -> None: + gold = _gold_table(2026081204, 8, 6, "joint_split_and_merge", "complex_correction") + prior, operations = _prior_for_edit(gold, "joint_split_and_merge", "complex_correction") + assert len(operations) >= 2 + assert not validate_cells(prior["cells"]) + + +@pytest.mark.parametrize("columns", [2, 3, 5, 14]) +def test_extra_split_is_non_identity_across_configured_column_range( + columns: int, +) -> None: + gold = _gold_table( + 2026081205 + columns, + 5, + columns, + "weak_borders extra_split", + "single_minimal_edit", + ) + prior, operations = _prior_for_edit( + gold, "extra_split", "single_minimal_edit" + ) + assert operations == ["split_spanning_cell"] + assert prior["semantic_state_sha256"] != gold["semantic_state_sha256"] + + +def test_incomplete_grid_is_rejection_fixture_only() -> None: + assert _negative_grid_fixture() == "GRID_INCOMPLETE" + + +def test_stable_json_hash_is_order_independent() -> None: + assert _sha256(_stable_json({"a": 1, "b": 2})) == _sha256( + _stable_json({"b": 2, "a": 1}) + ) + + +def test_counterfactual_pair_shares_edit_capable_gold_and_image() -> None: + root = Path(__file__).resolve().parents[1] + config = json.loads( + (root / "configs" / "generation_parameters_2026.08.12.4.json").read_text() + ) + schema = json.loads( + (root / "schemas" / "synthetic_table_record_2026.08.12.4.json").read_text() + ) + license_sha = _sha256(_stable_json(_license_manifest())) + shared = "weak_borders extra_split" + keep, keep_image = _make_record( + CoverageRequest("hard_keep", "KEEP", "weak_borders", "development"), + 0, + 0, + 0, + config, + license_sha, + schema, + shared, + ) + edit, edit_image = _make_record( + CoverageRequest( + "single_minimal_edit", "EDIT", "extra_split", "development" + ), + 1, + 0, + 0, + config, + license_sha, + schema, + shared, + ) + assert keep["gold"]["full_state_sha256"] == edit["gold"]["full_state_sha256"] + assert keep_image == edit_image + assert keep["gold"]["semantic_state_sha256"] == keep["prior"]["semantic_state_sha256"] + assert edit["gold"]["semantic_state_sha256"] != edit["prior"]["semantic_state_sha256"] + + +def test_single_cross_role_structure_signal_is_not_a_hard_overlap() -> None: + root = Path(__file__).resolve().parents[1] + config = json.loads( + (root / "configs" / "generation_parameters_2026.08.12.4.json").read_text() + ) + schema = json.loads( + (root / "schemas" / "synthetic_table_record_2026.08.12.4.json").read_text() + ) + license_sha = _sha256(_stable_json(_license_manifest())) + first, first_image = _make_record( + CoverageRequest("exact_keep", "KEEP", "ordinary_grid", "train"), + 0, + 1, + None, + config, + license_sha, + schema, + ) + second, second_image = _make_record( + CoverageRequest("exact_keep", "KEEP", "ordinary_grid", "holdout"), + 1, + 2, + None, + config, + license_sha, + schema, + ) + report = _overlap_report( + [first, second], + {first["sample_id"]: first_image, second["sample_id"]: second_image}, + ) + assert report["status"] == "PASS"