diff --git a/README.md b/README.md index 0a59629..d33bb8f 100644 --- a/README.md +++ b/README.md @@ -26,14 +26,15 @@ OmniDocBench is used as one document-parsing evaluation protocol. The methods and infrastructure in this repository are designed around the broader research problem of table structure recognition. -## Research snapshot 2026.08.12.4 +## Research snapshot 2026.08.12.5 -The `2026.08.12.4` snapshot establishes the shared representation, evaluation, -and synthetic-corpus specification for two independent modeling tracks. It +The `2026.08.12.5` snapshot establishes the shared representation, evaluation, +and frozen shared-corpus builder for two independent modeling tracks. It includes Canonical Table normalization, order-invariant topology targets, -candidate-integrity checks, table-only model interfaces, a frozen 256-record -generator-smoke design, and data-free regression tests. Model checkpoints and -dataset payloads are maintained outside this repository. +candidate-integrity checks, table-only model interfaces, a sealed 256-record +generator smoke, a preregistered 40,000-record corpus design, scalable overlap +audits, and data-free regression tests. Model checkpoints and dataset payloads +are maintained outside this repository. Project-authored releases follow calendar versioning: @@ -107,6 +108,7 @@ borderless-table-structuring-lab/ ├── scripts/ # Calendar-versioned corpus entry points ├── src/borderless_table_structuring/ │ ├── canonical.py # Canonical table normalization +│ ├── shared_corpus.py # Streaming shared-corpus build and audit │ ├── explicit.py # Public Explicit-route interface │ ├── candidate_interfaces.py │ ├── candidate_integrity.py @@ -145,7 +147,7 @@ compression, blur, background, and scanning artifacts. Dataset roles are assigned by document, template, content, renderer, and seed families before rendering. Exact and near-duplicate audits operate on images, text, normalized structure, geometry, and provenance. See the -[synthetic data specification](docs/corpus/SYNTHETIC_DATA_SPECIFICATION_2026.08.12.4.md) +[shared-corpus specification](docs/corpus/SHARED_CORPUS_SPECIFICATION_2026.08.12.md) and [data governance guide](docs/corpus/DATA_GOVERNANCE_2026.08.12.1.md). ## Installation @@ -174,6 +176,17 @@ python scripts/generate_data_smoke_2026.08.12.4.py \ --output /absolute/path/to/data-smoke-2026.08.12.4 ``` +After independently verifying that sealed smoke, build the preregistered +shared corpus into a different new external path: + +```bash +python scripts/build_shared_corpus_2026.08.12.py \ + --output /absolute/path/to/shared-corpus-2026.08.12 +``` + +The builder is CPU-first, streams the 40,000 records, and refuses an existing +output path. Use `--verify-only` for an independent payload and checksum pass. + ## Reproducible research Experiments record the code revision, schema release, immutable data revision, diff --git a/configs/shared_corpus_parameters_2026.08.12.json b/configs/shared_corpus_parameters_2026.08.12.json new file mode 100644 index 0000000..b0f8ac8 --- /dev/null +++ b/configs/shared_corpus_parameters_2026.08.12.json @@ -0,0 +1,62 @@ +{ + "configuration_release": "2026.08.12", + "dataset_release": "shared-corpus-2026.08.12", + "generator_release": "2026.08.12.5", + "schema_release": "synthetic-table-record-2026.08.12.5", + "seed_namespace": "borderless-table-structuring-shared-corpus-2026.08.12", + "seed_start": 202608120000000, + "requested_records": 40000, + "allow_failed_seed_replacement": false, + "gate_costs": { + "false_edit": 4.0, + "missed_edit": 1.0, + "uncertain_action": "KEEP" + }, + "categories": { + "exact_keep": {"count": 10000, "gate_label": "KEEP"}, + "hard_keep": {"count": 10000, "gate_label": "KEEP"}, + "single_minimal_edit": {"count": 10000, "gate_label": "EDIT"}, + "complex_correction": {"count": 10000, "gate_label": "EDIT"} + }, + "roles_per_category": { + "train": 7000, + "development": 2000, + "holdout": 1000 + }, + "counterfactual_pairs": { + "minimum_pairs": 10000, + "same_role_required": true, + "shared_fields": [ + "content_family_id", + "template_family_id", + "font_family_id", + "renderer_family_id", + "base_degradation" + ] + }, + "structure": { + "rows": {"minimum": 3, "maximum": 28}, + "columns": {"minimum": 2, "maximum": 14}, + "header_depth": {"minimum": 1, "maximum": 4}, + "maximum_cells": 240 + }, + "corruptions": { + "single_minimal_edit_exactly_one": true, + "complex_correction_minimum_operations": 2, + "complex_correction_maximum_operations": 4, + "accepted_prior_must_remain_canonical_legal": true, + "invalid_grid_fixtures_are_test_only": true + }, + "determinism": { + "semantic_replay_runs": 2, + "normalized_pixel_replay_runs": 2, + "sha256_algorithm": "sha256" + }, + "execution": { + "cpu_first": true, + "stream_records": true, + "stream_checksums": true, + "maximum_records_in_memory": 2, + "progress_interval_records": 500 + } +} diff --git a/docs/corpus/SHARED_CORPUS_COVERAGE_2026.08.12.csv b/docs/corpus/SHARED_CORPUS_COVERAGE_2026.08.12.csv new file mode 100644 index 0000000..a69dddf --- /dev/null +++ b/docs/corpus/SHARED_CORPUS_COVERAGE_2026.08.12.csv @@ -0,0 +1,30 @@ +category,gate_label,phenomenon,requested_count,train_count,development_count,holdout_count,counterfactual_pair_required,primary_risk +exact_keep,KEEP,simple_regular,1875,1312,375,188,false,false_edit +exact_keep,KEEP,hierarchical_header,1875,1313,375,187,false,false_edit +exact_keep,KEEP,row_or_column_span,1875,1312,375,188,false,false_edit +exact_keep,KEEP,empty_or_sparse_cells,1250,875,250,125,false,false_edit +exact_keep,KEEP,dense_or_multiline_text,1250,875,250,125,false,text_loss +exact_keep,KEEP,mixed_script_or_formula,937,656,187,94,false,text_loss +exact_keep,KEEP,clean_high_resolution_control,938,657,188,93,false,false_edit +hard_keep,KEEP,multilevel_irregular_header,1875,1312,375,188,true,false_edit +hard_keep,KEEP,mixed_two_dimensional_spans,1875,1313,375,187,true,false_edit +hard_keep,KEEP,weak_missing_or_partial_borders,1875,1312,375,188,true,false_edit +hard_keep,KEEP,empty_near_empty_or_sparse,1250,875,250,125,true,false_edit +hard_keep,KEEP,dense_small_long_or_multiline_text,1250,875,250,125,true,text_loss +hard_keep,KEEP,rotation_blur_compression_or_noise,1250,875,250,125,true,false_edit +hard_keep,KEEP,color_background_or_alignment_trap,625,438,125,62,true,false_edit +single_minimal_edit,EDIT,extra_split,1875,1312,375,188,true,cell_inflation +single_minimal_edit,EDIT,missing_split,1563,1094,313,156,true,under_segmentation +single_minimal_edit,EDIT,incorrect_merge,1562,1094,312,156,true,content_reassignment +single_minimal_edit,EDIT,missing_merge,1563,1094,313,156,true,over_segmentation +single_minimal_edit,EDIT,span_extent_error,1250,875,250,125,true,topology_error +single_minimal_edit,EDIT,token_ownership_error,937,656,187,94,true,text_loss +single_minimal_edit,EDIT,geometry_inconsistency,625,438,125,62,true,location_error +single_minimal_edit,EDIT,row_or_column_assignment_error,625,437,125,63,true,topology_error +complex_correction,EDIT,multilevel_header_with_spans,1875,1312,375,188,false,topology_error +complex_correction,EDIT,joint_split_and_merge,1875,1313,375,187,false,cell_inflation +complex_correction,EDIT,weak_border_with_topology_error,1563,1094,313,156,false,missed_edit +complex_correction,EDIT,empty_cells_with_spans,1250,875,250,125,false,ownership_error +complex_correction,EDIT,dense_multiline_with_topology_error,1250,875,250,125,false,text_loss +complex_correction,EDIT,mixed_script_formula_with_topology_error,937,656,187,94,false,text_loss +complex_correction,EDIT,rotation_noise_with_topology_error,1250,875,250,125,false,geometry_error diff --git a/docs/corpus/SHARED_CORPUS_SPECIFICATION_2026.08.12.md b/docs/corpus/SHARED_CORPUS_SPECIFICATION_2026.08.12.md new file mode 100644 index 0000000..7aeb1bc --- /dev/null +++ b/docs/corpus/SHARED_CORPUS_SPECIFICATION_2026.08.12.md @@ -0,0 +1,82 @@ +# Shared Corpus Specification 2026.08.12 + +Status: `FROZEN_FOR_NON_OVERWRITING_BUILD` + +Dataset identifier: `shared-corpus-2026.08.12` + +Generator identifier: `2026.08.12.5` + +Schema identifier: `synthetic-table-record-2026.08.12.5` + +## Research purpose + +This release builds a shared, source-traceable corpus for independent Explicit +Layout Transformer and LoRA Table Model research. It emphasizes conservative +KEEP behavior, direct final Canonical Table supervision, and minimal legal +structural correction without using terminal benchmark content. + +## Frozen scale and roles + +| Category | Train | Development | Holdout | Total | +|---|---:|---:|---:|---:| +| Exact KEEP | 7,000 | 2,000 | 1,000 | 10,000 | +| Hard KEEP | 7,000 | 2,000 | 1,000 | 10,000 | +| Single Minimal Edit | 7,000 | 2,000 | 1,000 | 10,000 | +| Complex Correction | 7,000 | 2,000 | 1,000 | 10,000 | +| **Total** | **28,000** | **8,000** | **4,000** | **40,000** | + +Hard KEEP and Single Minimal Edit form 10,000 same-role counterfactual pairs. +Each pair shares one rendered observation, one complete Gold state, and all +declared family identities. The KEEP member uses Gold unchanged. The EDIT +member uses one legal, non-identity Raw-like prior. + +## Supervision views + +- Gate: KEEP or EDIT, with false editing weighted four times a missed edit. +- Explicit: Raw-like prior, Raw text frozen, and an order-invariant structural + difference to Gold. +- LoRA: one complete table-only Canonical Table target. + +An arbitrary ordered merge/split program is not a primary target. + +## Source and license boundary + +All accepted content is project-authored synthetic content rendered with +rebuild-only runtime assets recorded in the source-license manifest. No +OmniDocBench or Customer50 page, crop, image, string, coordinate, annotation, +identifier, embedding, or transformed derivative may enter the build. + +## Isolation + +Role assignment occurs before rendering. Document, source, template, content, +renderer, font, and seed families are role-scoped. The build must report exact +payload/table matches, family overlaps, perceptual near matches with supporting +text or geometry evidence, and normalized structure/text/geometry signature +statistics. Any cross-role exact or unresolved near overlap is a hard stop. + +## Execution + +The generator is CPU-first and streams records, images, checksums, and compact +audit signatures. It must never retain more than two full records in memory. +The output path is new and non-overwriting. Failed seeds are not replaced. + +## Acceptance + +- Requested, generated, and accepted: 40,000. +- Quarantined and failed: 0. +- Exact KEEP / Hard KEEP / Single / Complex: 10,000 each. +- Train / development / holdout: 28,000 / 8,000 / 4,000. +- Complete same-role counterfactual pairs: 10,000. +- Schema, Canonical legality, complete grid, token ownership, geometry, + deterministic semantic replay, and deterministic normalized-pixel replay: + 40,000/40,000. +- KEEP identity: 20,000/20,000. +- EDIT legal non-identity replay: 20,000/20,000. +- Cross-role overlap audit: PASS. +- Terminal inputs used: false. +- Complete SHA256 manifest and independent verification: PASS. + +## Boundary + +A sealed corpus authorizes bounded, preregistered candidate pilots only. It +does not authorize full model training. diff --git a/pyproject.toml b/pyproject.toml index 40ab10f..0208e4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "borderless-table-structuring-lab" -version = "2026.8.12.4" +version = "2026.8.12.5" description = "Research on canonical representations, explicit topology modeling, and generative adaptation for borderless table structure recognition." readme = "README.md" requires-python = ">=3.10" diff --git a/schemas/synthetic_table_record_2026.08.12.5.json b/schemas/synthetic_table_record_2026.08.12.5.json new file mode 100644 index 0000000..84585bb --- /dev/null +++ b/schemas/synthetic_table_record_2026.08.12.5.json @@ -0,0 +1,91 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "borderless-table-structuring/synthetic-table-record/2026.08.12.5", + "title": "Synthetic Table Record 2026.08.12.5", + "type": "object", + "required": [ + "schema_release", + "dataset_release", + "sample_id", + "role", + "gate_label", + "category", + "identity", + "provenance", + "rendering", + "payloads", + "gold", + "prior", + "views", + "audits" + ], + "properties": { + "schema_release": {"const": "synthetic-table-record-2026.08.12.5"}, + "dataset_release": {"const": "shared-corpus-2026.08.12"}, + "sample_id": {"type": "string", "pattern": "^shared-corpus-2026\\.08\\.12-[0-9]{6}$"}, + "role": {"enum": ["train", "development", "holdout"]}, + "gate_label": {"enum": ["KEEP", "EDIT", "QUARANTINE"]}, + "category": {"enum": ["exact_keep", "hard_keep", "single_minimal_edit", "complex_correction"]}, + "identity": { + "type": "object", + "required": ["generation_seed", "document_cluster_id", "source_family_id", "template_family_id", "content_family_id", "renderer_family_id", "font_family_id", "counterfactual_pair_id"], + "properties": { + "generation_seed": {"type": "integer", "minimum": 0}, + "document_cluster_id": {"type": "string", "minLength": 1}, + "source_family_id": {"type": "string", "minLength": 1}, + "template_family_id": {"type": "string", "minLength": 1}, + "content_family_id": {"type": "string", "minLength": 1}, + "renderer_family_id": {"type": "string", "minLength": 1}, + "font_family_id": {"type": "string", "minLength": 1}, + "counterfactual_pair_id": {"type": ["string", "null"]} + }, + "additionalProperties": false + }, + "provenance": { + "type": "object", + "required": ["generator_release", "source_id", "license_decision", "license_manifest_sha256", "terminal_inputs_used"], + "properties": { + "generator_release": {"const": "2026.08.12.5"}, + "source_id": {"type": "string", "minLength": 1}, + "license_decision": {"enum": ["APPROVED_REDISTRIBUTABLE", "APPROVED_REBUILD_ONLY", "QUARANTINED"]}, + "license_manifest_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "terminal_inputs_used": {"const": false} + }, + "additionalProperties": true + }, + "rendering": {"type": "object", "required": ["parameters", "font_sources", "normalized_pixel_sha256"]}, + "payloads": {"type": "object", "required": ["image_path", "image_sha256", "record_sha256"]}, + "gold": {"$ref": "#/$defs/canonicalTable"}, + "prior": {"$ref": "#/$defs/canonicalTable"}, + "views": { + "type": "object", + "required": ["gate", "explicit", "lora"], + "properties": { + "gate": {"type": "object", "required": ["label", "false_edit_cost", "missed_edit_cost"]}, + "explicit": {"type": "object", "required": ["prior_state_sha256", "order_invariant_difference", "raw_text_frozen"]}, + "lora": {"type": "object", "required": ["complete_canonical_target", "table_only"]} + }, + "additionalProperties": false + }, + "audits": { + "type": "object", + "required": ["schema_valid", "canonical_legal", "prior_canonical_legal", "grid_complete", "prior_grid_complete", "gold_recompiled", "token_ownership_valid", "geometry_valid", "semantic_replay_valid", "normalized_pixel_replay_valid", "cross_role_overlap_clear"] + } + }, + "$defs": { + "canonicalTable": { + "type": "object", + "required": ["rows", "columns", "cells", "tokens", "semantic_state_sha256", "full_state_sha256"], + "properties": { + "rows": {"type": "integer", "minimum": 1}, + "columns": {"type": "integer", "minimum": 1}, + "cells": {"type": "array", "minItems": 1}, + "tokens": {"type": "array"}, + "semantic_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "full_state_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + }, + "additionalProperties": true + } + }, + "additionalProperties": false +} diff --git a/scripts/build_shared_corpus_2026.08.12.py b/scripts/build_shared_corpus_2026.08.12.py new file mode 100644 index 0000000..5e5a425 --- /dev/null +++ b/scripts/build_shared_corpus_2026.08.12.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import argparse +import json +import traceback +from pathlib import Path + +from borderless_table_structuring.shared_corpus import ( + build_shared_corpus, + verify_shared_corpus, +) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Build or verify the frozen shared corpus 2026.08.12." + ) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--verify-only", action="store_true") + parser.add_argument("--expected-records", type=int, default=40000) + parser.add_argument( + "--config", + type=Path, + default=Path("configs/shared_corpus_parameters_2026.08.12.json"), + ) + parser.add_argument( + "--coverage", + type=Path, + default=Path("docs/corpus/SHARED_CORPUS_COVERAGE_2026.08.12.csv"), + ) + parser.add_argument( + "--schema", + type=Path, + default=Path("schemas/synthetic_table_record_2026.08.12.5.json"), + ) + args = parser.parse_args() + try: + if args.verify_only: + result = verify_shared_corpus( + args.output, args.schema, expected_records=args.expected_records + ) + else: + result = build_shared_corpus( + args.output, args.config, args.coverage, args.schema + ) + except Exception as error: + if args.output.exists() and (args.output / "reports").is_dir(): + failure = { + "dataset_release": "shared-corpus-2026.08.12", + "generator_release": "2026.08.12.5", + "error_type": type(error).__name__, + "error": str(error), + "training": False, + "terminal_inputs_used": False, + "traceback": traceback.format_exc(), + } + (args.output / "reports" / "FAILURE_REPORT.json").write_text( + json.dumps(failure, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + raise + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/borderless_table_structuring/shared_corpus.py b/src/borderless_table_structuring/shared_corpus.py new file mode 100644 index 0000000..471f49f --- /dev/null +++ b/src/borderless_table_structuring/shared_corpus.py @@ -0,0 +1,498 @@ +from __future__ import annotations + +import copy +import hashlib +import json +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + +from jsonschema import Draft202012Validator + +from .synthetic_data import ( + CoverageRequest, + _expand_coverage, + _hamming, + _license_manifest, + _make_record, + _perceptual_hash, + _sha256, + _stable_json, + _validate_record_contract, +) + + +GENERATOR_RELEASE = "2026.08.12.5" +DATASET_RELEASE = "shared-corpus-2026.08.12" +SCHEMA_RELEASE = "synthetic-table-record-2026.08.12.5" +ZERO_SHA256 = "0" * 64 + + +def _prepare_assignments( + requests: list[CoverageRequest], +) -> dict[int, tuple[int, int | None, str | None]]: + grouped: dict[str, list[CoverageRequest]] = defaultdict(list) + for request in requests: + grouped[request.category].append(request) + assignments: dict[int, tuple[int, int | None, str | None]] = {} + base_family_index = 0 + hard = grouped["hard_keep"] + single = grouped["single_minimal_edit"] + if len(hard) != len(single): + raise ValueError("counterfactual category counts do not match") + for pair_index, (keep_request, edit_request) in enumerate(zip(hard, single)): + if keep_request.role != edit_request.role: + raise ValueError("counterfactual pair roles do not match") + shared = f"{keep_request.phenomenon} {edit_request.phenomenon}" + assignments[id(keep_request)] = (base_family_index, pair_index, shared) + assignments[id(edit_request)] = (base_family_index, pair_index, shared) + base_family_index += 1 + for category in ("exact_keep", "complex_correction"): + for request in grouped[category]: + assignments[id(request)] = (base_family_index, None, None) + base_family_index += 1 + return assignments + + +def _compact_signature(record: dict[str, Any], image: bytes) -> dict[str, Any]: + gold = record["gold"] + structure = { + "rows": gold["rows"], + "columns": gold["columns"], + "cells": [ + { + "row": cell["row"], + "col": cell["col"], + "rowspan": cell["rowspan"], + "colspan": cell["colspan"], + "tag": cell["tag"], + } + for cell in gold["cells"] + ], + } + structure_sha256 = _sha256(_stable_json(structure)) + text_sha256 = _sha256( + _stable_json([token["text"] for token in gold["tokens"]]) + ) + geometry_sha256 = _sha256( + _stable_json([cell["bbox"] for cell in gold["cells"]]) + ) + return { + "sample_id": record["sample_id"], + "role": record["role"], + "image_sha256": record["payloads"]["image_sha256"], + "pixel_sha256": record["rendering"]["normalized_pixel_sha256"], + "perceptual_hash": _perceptual_hash(image), + "structure_sha256": structure_sha256, + "text_sha256": text_sha256, + "geometry_sha256": geometry_sha256, + "table_sha256": _sha256( + _stable_json( + { + "structure": structure_sha256, + "text": text_sha256, + "geometry": geometry_sha256, + } + ) + ), + "families": { + key: record["identity"][key] + for key in ( + "document_cluster_id", + "source_family_id", + "template_family_id", + "content_family_id", + "renderer_family_id", + ) + }, + } + + +def _audit_signatures(signatures: list[dict[str, Any]]) -> dict[str, Any]: + exact: list[dict[str, Any]] = [] + near: list[dict[str, Any]] = [] + fields = ( + "image_sha256", + "pixel_sha256", + "table_sha256", + "structure_sha256", + "text_sha256", + "geometry_sha256", + ) + duplicate_statistics: dict[str, int] = {} + for field in fields: + buckets: dict[str, list[dict[str, Any]]] = defaultdict(list) + for item in signatures: + buckets[item[field]].append(item) + duplicate_statistics[field] = sum( + len(members) for members in buckets.values() if len(members) > 1 + ) + for value, members in buckets.items(): + roles = sorted({member["role"] for member in members}) + if len(roles) > 1 and field in ( + "image_sha256", + "pixel_sha256", + "table_sha256", + ): + exact.append( + { + "signal": field, + "value": value, + "roles": roles, + "sample_ids": [member["sample_id"] for member in members[:8]], + } + ) + for family_key in next(iter(signatures))["families"] if signatures else (): + buckets: dict[str, set[str]] = defaultdict(set) + for item in signatures: + buckets[item["families"][family_key]].add(item["role"]) + for value, roles in buckets.items(): + if len(roles) > 1: + exact.append( + { + "signal": family_key, + "value": value, + "roles": sorted(roles), + } + ) + # The near-overlap policy requires either identical normalized text or an + # identical structure-and-geometry pair. Blocking on those required + # supporting signals is exhaustive and avoids an unsafe all-pairs scan. + supporting_buckets: dict[tuple[str, ...], list[dict[str, Any]]] = defaultdict(list) + for item in signatures: + supporting_buckets[("text", item["text_sha256"])].append(item) + supporting_buckets[ + ("structure_geometry", item["structure_sha256"], item["geometry_sha256"]) + ].append(item) + seen_near_pairs: set[tuple[str, str]] = set() + for members in supporting_buckets.values(): + by_role: dict[str, list[dict[str, Any]]] = defaultdict(list) + for member in members: + by_role[member["role"]].append(member) + roles = sorted(by_role) + for role_index, first_role in enumerate(roles): + for second_role in roles[role_index + 1 :]: + for first in by_role[first_role]: + for second in by_role[second_role]: + pair = tuple(sorted((first["sample_id"], second["sample_id"]))) + if pair in seen_near_pairs: + continue + distance = _hamming( + first["perceptual_hash"], second["perceptual_hash"] + ) + supporting = [ + field + for field in ( + "text_sha256", + "structure_sha256", + "geometry_sha256", + ) + if first[field] == second[field] + ] + if distance <= 2 and ( + "text_sha256" in supporting + or all( + field in supporting + for field in ( + "structure_sha256", + "geometry_sha256", + ) + ) + ): + seen_near_pairs.add(pair) + near.append( + { + "first": first["sample_id"], + "second": second["sample_id"], + "distance": distance, + "supporting_signals": supporting, + } + ) + return { + "release": GENERATOR_RELEASE, + "dataset_release": DATASET_RELEASE, + "signature_count": len(signatures), + "cross_role_exact_overlaps": exact, + "cross_role_unresolved_near_overlaps": near, + "within_and_cross_role_duplicate_signature_members": duplicate_statistics, + "status": "PASS" if not exact and not near else "FAIL", + } + + +def build_shared_corpus( + output: Path, + config_path: Path, + coverage_path: Path, + schema_path: Path, +) -> dict[str, Any]: + if output.exists(): + raise FileExistsError(f"non-overwrite path already exists: {output}") + config = json.loads(config_path.read_text(encoding="utf-8")) + schema = json.loads(schema_path.read_text(encoding="utf-8")) + requests = _expand_coverage(coverage_path) + if len(requests) != config["requested_records"]: + raise ValueError("coverage count does not match configuration") + output.mkdir(parents=True) + for name in ("images", "manifests", "records", "audits", "reports", "quarantine"): + (output / name).mkdir() + + input_hashes = { + "config_sha256": _sha256(config_path.read_bytes()), + "coverage_sha256": _sha256(coverage_path.read_bytes()), + "schema_sha256": _sha256(schema_path.read_bytes()), + } + requested_records = int(config["requested_records"]) + (output / "PREREGISTRATION.md").write_text( + "# Shared Corpus Build Preregistration 2026.08.12\n\n" + f"This CPU-first run requests exactly {requested_records:,} new synthetic records. " + "It performs no model training and no benchmark evaluation. Every " + "record must pass the frozen schema, Canonical, replay, ownership, " + "geometry, source/license, pairing, isolation, and checksum gates. " + "A failed seed is not replaced.\n", + encoding="utf-8", + ) + run_config = { + "release": GENERATOR_RELEASE, + "dataset_release": DATASET_RELEASE, + "purpose": "FROZEN_SHARED_CORPUS_BUILD", + "training": False, + "benchmark_evaluation": False, + "terminal_inputs_used": False, + "output_path": str(output.resolve()), + **input_hashes, + } + (output / "run_config.json").write_text( + json.dumps(run_config, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (output / "input_manifest.json").write_text( + json.dumps( + { + "release": GENERATOR_RELEASE, + "inputs": [ + {"path": str(config_path), "sha256": input_hashes["config_sha256"]}, + {"path": str(coverage_path), "sha256": input_hashes["coverage_sha256"]}, + {"path": str(schema_path), "sha256": input_hashes["schema_sha256"]}, + ], + "terminal_inputs": [], + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + license_manifest = _license_manifest() + license_manifest["release"] = GENERATOR_RELEASE + license_bytes = ( + json.dumps(license_manifest, ensure_ascii=False, indent=2, sort_keys=True) + + "\n" + ).encode("utf-8") + license_sha = _sha256(license_bytes) + (output / "manifests" / "source_license.json").write_bytes(license_bytes) + + assignments = _prepare_assignments(requests) + validator = Draft202012Validator(schema) + category_counts: Counter[str] = Counter() + role_counts: Counter[str] = Counter() + pair_members: dict[str, dict[str, Any]] = {} + signatures: list[dict[str, Any]] = [] + manifest_hasher = hashlib.sha256() + role_hasher = hashlib.sha256() + manifest_path = output / "records" / "manifest.jsonl" + role_path = output / "manifests" / "role_assignment.jsonl" + with manifest_path.open("wb") as manifest, role_path.open("wb") as role_file: + for sample_index, request in enumerate(requests): + family_index, pair_index, shared = assignments[id(request)] + record, image = _make_record( + request, + sample_index, + family_index, + pair_index, + config, + license_sha, + schema, + shared, + dataset_release=DATASET_RELEASE, + schema_release=SCHEMA_RELEASE, + generator_release=GENERATOR_RELEASE, + shape_shuffle_seed=2026081205, + ) + replay, replay_image = _make_record( + request, + sample_index, + family_index, + pair_index, + config, + license_sha, + schema, + shared, + dataset_release=DATASET_RELEASE, + schema_release=SCHEMA_RELEASE, + generator_release=GENERATOR_RELEASE, + shape_shuffle_seed=2026081205, + ) + if record != replay: + raise ValueError(f"NONDETERMINISTIC_SEMANTICS: {record['sample_id']}") + if image != replay_image: + raise ValueError(f"NONDETERMINISTIC_PIXELS: {record['sample_id']}") + _validate_record_contract(record) + (output / record["payloads"]["image_path"]).write_bytes(image) + line = json.dumps(record, ensure_ascii=False, sort_keys=True).encode("utf-8") + b"\n" + manifest.write(line) + manifest_hasher.update(line) + role_line = ( + json.dumps( + { + "sample_id": record["sample_id"], + "role": record["role"], + "template_family_id": record["identity"]["template_family_id"], + "content_family_id": record["identity"]["content_family_id"], + "renderer_family_id": record["identity"]["renderer_family_id"], + "counterfactual_pair_id": record["identity"]["counterfactual_pair_id"], + }, + sort_keys=True, + ).encode("utf-8") + + b"\n" + ) + role_file.write(role_line) + role_hasher.update(role_line) + category_counts[record["category"]] += 1 + role_counts[record["role"]] += 1 + signatures.append(_compact_signature(record, image)) + pair_id = record["identity"]["counterfactual_pair_id"] + if pair_id is not None: + state = pair_members.setdefault( + pair_id, + { + "count": 0, + "role": record["role"], + "gold": record["gold"]["full_state_sha256"], + "image": record["payloads"]["image_sha256"], + "valid": True, + }, + ) + state["count"] += 1 + state["valid"] = state["valid"] and all( + ( + state["role"] == record["role"], + state["gold"] == record["gold"]["full_state_sha256"], + state["image"] == record["payloads"]["image_sha256"], + ) + ) + if (sample_index + 1) % config["execution"]["progress_interval_records"] == 0: + (output / "reports" / "progress.json").write_text( + json.dumps( + { + "release": GENERATOR_RELEASE, + "dataset_release": DATASET_RELEASE, + "generated": sample_index + 1, + "requested": len(requests), + "failed": 0, + "training": False, + "terminal_inputs_used": False, + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + overlap = _audit_signatures(signatures) + (output / "audits" / "overlap_report.json").write_text( + json.dumps(overlap, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (output / "audits" / "signatures.jsonl").write_text( + "".join(json.dumps(item, sort_keys=True) + "\n" for item in signatures), + encoding="utf-8", + ) + complete_pairs = sum( + state["count"] == 2 and state["valid"] for state in pair_members.values() + ) + expected_categories = { + key: value["count"] for key, value in config["categories"].items() + } + expected_roles = { + role: count * 4 for role, count in config["roles_per_category"].items() + } + checks = { + "count": sum(category_counts.values()) == config["requested_records"], + "categories": dict(category_counts) == expected_categories, + "roles": dict(role_counts) == expected_roles, + "pairs": complete_pairs >= config["counterfactual_pairs"]["minimum_pairs"], + "overlap": overlap["status"] == "PASS", + "terminal_nonuse": True, + } + acceptance = { + "release": GENERATOR_RELEASE, + "dataset_release": DATASET_RELEASE, + "requested": len(requests), + "generated": sum(category_counts.values()), + "accepted": sum(category_counts.values()), + "quarantined": 0, + "failed": 0, + "category_counts": dict(sorted(category_counts.items())), + "role_counts": dict(sorted(role_counts.items())), + "complete_counterfactual_pairs": complete_pairs, + "manifest_sha256": manifest_hasher.hexdigest(), + "role_assignment_sha256": role_hasher.hexdigest(), + "overlap_status": overlap["status"], + "terminal_inputs_used": False, + "checks": checks, + "status": "PASS" if all(checks.values()) else "FAIL", + } + (output / "reports" / "acceptance_report.json").write_text( + json.dumps(acceptance, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + (output / "quarantine" / "quarantine.jsonl").write_text("", encoding="utf-8") + if acceptance["status"] != "PASS": + raise RuntimeError(f"shared corpus build failed: {checks}") + files = sorted( + path + for path in output.rglob("*") + if path.is_file() and path.name != "SHA256SUMS" + ) + (output / "SHA256SUMS").write_text( + "".join( + f"{_sha256(path.read_bytes())} {path.relative_to(output)}\n" + for path in files + ), + encoding="utf-8", + ) + return acceptance + + +def verify_shared_corpus( + output: Path, schema_path: Path, expected_records: int = 40000 +) -> dict[str, Any]: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + validator = Draft202012Validator(schema) + count = 0 + with (output / "records" / "manifest.jsonl").open(encoding="utf-8") as handle: + for line in handle: + if not line.strip(): + continue + record = json.loads(line) + validator.validate(record) + _validate_record_contract(record) + hashed = copy.deepcopy(record) + expected_record_sha = hashed["payloads"]["record_sha256"] + hashed["payloads"]["record_sha256"] = ZERO_SHA256 + if _sha256(_stable_json(hashed)) != expected_record_sha: + raise ValueError(f"record hash mismatch: {record['sample_id']}") + image_path = output / record["payloads"]["image_path"] + if _sha256(image_path.read_bytes()) != record["payloads"]["image_sha256"]: + raise ValueError(f"image hash mismatch: {record['sample_id']}") + count += 1 + if count != expected_records: + raise ValueError(f"record count mismatch: {count} != {expected_records}") + for line in (output / "SHA256SUMS").read_text(encoding="utf-8").splitlines(): + expected_sha, relative_path = line.split(" ", 1) + if _sha256((output / relative_path).read_bytes()) != expected_sha: + raise ValueError(f"sealed payload hash mismatch: {relative_path}") + acceptance = json.loads( + (output / "reports" / "acceptance_report.json").read_text(encoding="utf-8") + ) + if acceptance["status"] != "PASS": + raise ValueError("shared corpus acceptance is not PASS") + return acceptance diff --git a/src/borderless_table_structuring/synthetic_data.py b/src/borderless_table_structuring/synthetic_data.py index bf66934..23d0f80 100644 --- a/src/borderless_table_structuring/synthetic_data.py +++ b/src/borderless_table_structuring/synthetic_data.py @@ -7,6 +7,7 @@ import random from collections import Counter, defaultdict from dataclasses import dataclass +from functools import lru_cache from io import BytesIO from pathlib import Path from typing import Any, Iterable @@ -377,10 +378,11 @@ def _expand_coverage(path: Path) -> list[CoverageRequest]: return sorted(requests, key=lambda item: (order[item.category], item.role, item.phenomenon)) -def _shape_pool() -> list[tuple[int, int]]: +@lru_cache(maxsize=None) +def _shape_pool(shuffle_seed: int = 2026081204) -> tuple[tuple[int, int], ...]: values = [(rows, columns) for rows in range(3, 29) for columns in range(2, 15) if rows * columns <= 240] - random.Random(2026081204).shuffle(values) - return values + random.Random(shuffle_seed).shuffle(values) + return tuple(values) def _make_record( @@ -392,10 +394,15 @@ def _make_record( license_manifest_sha256: str, schema: dict[str, Any], shared_phenomenon: str | None = None, + dataset_release: str = DATASET_RELEASE, + schema_release: str = SCHEMA_RELEASE, + generator_release: str = RELEASE, + shape_shuffle_seed: int = 2026081204, ) -> tuple[dict[str, Any], bytes]: seed = int(config["seed_start"]) + sample_index base_seed = int(config["seed_start"]) + 100000 + base_family_index - rows, columns = _shape_pool()[base_family_index] + shape_pool = _shape_pool(shape_shuffle_seed) + rows, columns = shape_pool[base_family_index % len(shape_pool)] pair_id = f"pair-{pair_index:06d}" if pair_index is not None else None family = pair_id or f"family-{base_family_index:06d}" rendering_phenomenon = shared_phenomenon or request.phenomenon @@ -412,11 +419,11 @@ def _make_record( prior, operations = _prior_for_edit(gold, request.phenomenon, request.category) difference = _difference(prior, gold) image_bytes, pixel_sha, render_parameters = _draw_table(gold, rendering_phenomenon, base_seed) - image_name = f"images/{DATASET_RELEASE}-{sample_index:06d}.png" + image_name = f"images/{dataset_release}-{sample_index:06d}.png" record = { - "schema_release": SCHEMA_RELEASE, - "dataset_release": DATASET_RELEASE, - "sample_id": f"{DATASET_RELEASE}-{sample_index:06d}", + "schema_release": schema_release, + "dataset_release": dataset_release, + "sample_id": f"{dataset_release}-{sample_index:06d}", "role": request.role, "gate_label": request.gate_label, "category": request.category, @@ -431,7 +438,7 @@ def _make_record( "counterfactual_pair_id": pair_id, }, "provenance": { - "generator_release": RELEASE, + "generator_release": generator_release, "source_id": "project-authored-synthetic-content", "license_decision": LICENSE_DECISION, "license_manifest_sha256": license_manifest_sha256, diff --git a/tests/test_shared_corpus_2026_08_12.py b/tests/test_shared_corpus_2026_08_12.py new file mode 100644 index 0000000..f31f23d --- /dev/null +++ b/tests/test_shared_corpus_2026_08_12.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import csv +import json +from collections import Counter, defaultdict +from pathlib import Path + +from borderless_table_structuring.shared_corpus import ( + DATASET_RELEASE, + GENERATOR_RELEASE, + SCHEMA_RELEASE, + _audit_signatures, + _prepare_assignments, + build_shared_corpus, + verify_shared_corpus, +) +from borderless_table_structuring.synthetic_data import ( + CoverageRequest, + _expand_coverage, + _shape_pool, +) + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_frozen_shared_corpus_distribution() -> None: + config = json.loads( + (ROOT / "configs" / "shared_corpus_parameters_2026.08.12.json").read_text() + ) + requests = _expand_coverage( + ROOT / "docs" / "corpus" / "SHARED_CORPUS_COVERAGE_2026.08.12.csv" + ) + categories = Counter(request.category for request in requests) + roles = Counter(request.role for request in requests) + category_roles: dict[str, Counter[str]] = defaultdict(Counter) + for request in requests: + category_roles[request.category][request.role] += 1 + + assert len(requests) == config["requested_records"] == 40000 + assert categories == Counter( + { + "exact_keep": 10000, + "hard_keep": 10000, + "single_minimal_edit": 10000, + "complex_correction": 10000, + } + ) + assert roles == Counter({"train": 28000, "development": 8000, "holdout": 4000}) + assert all( + counts == Counter({"train": 7000, "development": 2000, "holdout": 1000}) + for counts in category_roles.values() + ) + + +def test_frozen_schema_and_config_identifiers_agree() -> None: + config = json.loads( + (ROOT / "configs" / "shared_corpus_parameters_2026.08.12.json").read_text() + ) + schema = json.loads( + (ROOT / "schemas" / "synthetic_table_record_2026.08.12.5.json").read_text() + ) + assert config["dataset_release"] == DATASET_RELEASE + assert config["generator_release"] == GENERATOR_RELEASE + assert config["schema_release"] == SCHEMA_RELEASE + assert schema["properties"]["dataset_release"]["const"] == DATASET_RELEASE + assert schema["properties"]["schema_release"]["const"] == SCHEMA_RELEASE + assert schema["properties"]["provenance"]["properties"]["generator_release"]["const"] == GENERATOR_RELEASE + + +def test_counterfactual_assignments_are_complete_and_same_role() -> None: + requests = _expand_coverage( + ROOT / "docs" / "corpus" / "SHARED_CORPUS_COVERAGE_2026.08.12.csv" + ) + assignments = _prepare_assignments(requests) + paired: dict[int, list[CoverageRequest]] = defaultdict(list) + for request in requests: + family_index, pair_index, shared = assignments[id(request)] + assert family_index >= 0 + if pair_index is not None: + assert shared + paired[pair_index].append(request) + assert len(assignments) == len(requests) + assert len(paired) == 10000 + assert all( + len(members) == 2 + and {member.category for member in members} + == {"hard_keep", "single_minimal_edit"} + and len({member.role for member in members}) == 1 + for members in paired.values() + ) + + +def test_shape_pool_is_cached_and_supports_reuse() -> None: + first = _shape_pool(2026081205) + second = _shape_pool(2026081205) + assert first is second + assert len(first) > 100 + assert all(3 <= rows <= 28 and 2 <= columns <= 14 for rows, columns in first) + assert all(rows * columns <= 240 for rows, columns in first) + + +def _signature(sample: str, role: str, phash: str, *, text: str, structure: str, geometry: str) -> dict[str, object]: + return { + "sample_id": sample, + "role": role, + "image_sha256": f"image-{sample}", + "pixel_sha256": f"pixel-{sample}", + "table_sha256": f"table-{sample}", + "structure_sha256": structure, + "text_sha256": text, + "geometry_sha256": geometry, + "perceptual_hash": phash, + "families": { + "document_cluster_id": f"document-{role}-{sample}", + "source_family_id": f"source-{role}-{sample}", + "template_family_id": f"template-{role}-{sample}", + "content_family_id": f"content-{role}-{sample}", + "renderer_family_id": f"renderer-{role}-{sample}", + }, + } + + +def test_near_overlap_audit_is_not_limited_to_a_phash_prefix() -> None: + # These hashes differ in the first hexadecimal digit but have Hamming + # distance one. Matching text is the preregistered supporting signal. + first = _signature( + "first", "train", "0000000000000000", text="same", structure="a", geometry="b" + ) + second = _signature( + "second", "holdout", "8000000000000000", text="same", structure="c", geometry="d" + ) + report = _audit_signatures([first, second]) + assert report["status"] == "FAIL" + assert report["cross_role_unresolved_near_overlaps"][0]["distance"] == 1 + + +def test_single_structure_signature_is_reported_but_not_rejected() -> None: + first = _signature( + "first", "train", "0000000000000000", text="a", structure="same", geometry="b" + ) + second = _signature( + "second", "holdout", "ffffffffffffffff", text="c", structure="same", geometry="d" + ) + report = _audit_signatures([first, second]) + assert report["status"] == "PASS" + assert report["within_and_cross_role_duplicate_signature_members"]["structure_sha256"] == 2 + + +def test_small_non_overwriting_build_and_independent_verification(tmp_path: Path) -> None: + config = json.loads( + (ROOT / "configs" / "shared_corpus_parameters_2026.08.12.json").read_text() + ) + config["requested_records"] = 4 + config["categories"] = { + "exact_keep": {"count": 1, "gate_label": "KEEP"}, + "hard_keep": {"count": 1, "gate_label": "KEEP"}, + "single_minimal_edit": {"count": 1, "gate_label": "EDIT"}, + "complex_correction": {"count": 1, "gate_label": "EDIT"}, + } + config["roles_per_category"] = {"development": 1} + config["counterfactual_pairs"]["minimum_pairs"] = 1 + config["execution"]["progress_interval_records"] = 1 + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + coverage_path = tmp_path / "coverage.csv" + with coverage_path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter( + handle, + fieldnames=[ + "category", + "gate_label", + "phenomenon", + "requested_count", + "train_count", + "development_count", + "holdout_count", + "counterfactual_pair_required", + "primary_risk", + ], + ) + writer.writeheader() + for category, label, phenomenon in ( + ("exact_keep", "KEEP", "simple_regular"), + ("hard_keep", "KEEP", "weak_borders"), + ("single_minimal_edit", "EDIT", "extra_split"), + ("complex_correction", "EDIT", "joint_split_and_merge"), + ): + writer.writerow( + { + "category": category, + "gate_label": label, + "phenomenon": phenomenon, + "requested_count": 1, + "train_count": 0, + "development_count": 1, + "holdout_count": 0, + "counterfactual_pair_required": str( + category in {"hard_keep", "single_minimal_edit"} + ).lower(), + "primary_risk": "bounded_test", + } + ) + schema_path = ROOT / "schemas" / "synthetic_table_record_2026.08.12.5.json" + output = tmp_path / "shared-corpus-test" + acceptance = build_shared_corpus(output, config_path, coverage_path, schema_path) + verified = verify_shared_corpus(output, schema_path, expected_records=4) + assert acceptance["status"] == verified["status"] == "PASS" + assert acceptance["complete_counterfactual_pairs"] == 1 + assert (output / "SHA256SUMS").is_file()