From bb9ca4c0d4ed634edf053c951d9ce5f05f97f28c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 20:44:04 +0200 Subject: [PATCH 01/11] Add frontend bundle verification to the evaluation harness verify_frontend_bundle checks a published cross_dataset.frontend_bundle.v1 directory the way the web reader does: schema version, safe partition paths, recorded SHA-256 for every listed partition, page sequence and count reconciliation, and that each partition carries the manifest's run and snapshot IDs. frontend_bundle_partitions exposes the descriptors in a stable order so publishers can share the same attested file list. Co-Authored-By: Claude Fable 5 --- evaluation_harness/frontend_bundle.py | 113 ++++++++++++++++++++++++++ tests/test_frontend_bundle.py | 51 ++++++++++++ 2 files changed, 164 insertions(+) diff --git a/evaluation_harness/frontend_bundle.py b/evaluation_harness/frontend_bundle.py index 75a9603..4d167b0 100644 --- a/evaluation_harness/frontend_bundle.py +++ b/evaluation_harness/frontend_bundle.py @@ -3,6 +3,7 @@ import hashlib import json import math +import re from collections import Counter from decimal import Decimal from pathlib import Path @@ -21,6 +22,10 @@ FRONTEND_BUNDLE_SCHEMA = "cross_dataset.frontend_bundle.v1" WITHIN_BOUNDS_RELATIVE_ERROR = Decimal("0.10") FAR_OUTSIDE_BOUNDS_RELATIVE_ERROR = Decimal("0.25") +# Mirrors the web reader's partition descriptor rules (frontend/lib/cross-dataset/ +# artifact.ts) so a bundle that passes here is one the dashboard will accept. +_PARTITION_PATH = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]*$") +_SHA256_HEX = re.compile(r"^[a-f0-9]{64}$") DEFAULT_SOURCE_LABELS = { "census_acs_pums_2024": "Raw ACS PUMS", @@ -808,3 +813,111 @@ def add_facet(dimension: str, key: str, page: int) -> None: } (output / "manifest.json").write_bytes(_document(manifest)) return manifest + + +def _partition_descriptor(value: Any, field: str) -> dict[str, Any]: + if not isinstance(value, dict): + raise ValueError(f"frontend bundle partition descriptor {field} is missing") + path = value.get("path") + sha256 = value.get("sha256") + if not isinstance(path, str) or not path: + raise ValueError(f"frontend bundle partition {field} has no path") + if ( + path.startswith("/") + or "\\" in path + or any(part in ("", "..") for part in path.split("/")) + or not _PARTITION_PATH.match(path) + ): + raise ValueError( + f"frontend bundle partition {field} has an unsafe path: {path!r}" + ) + if not isinstance(sha256, str) or not _SHA256_HEX.match(sha256): + raise ValueError(f"frontend bundle partition {field} has no SHA-256") + return value + + +def frontend_bundle_partitions(manifest: dict[str, Any]) -> list[dict[str, Any]]: + """Return the manifest's partition descriptors in a stable order. + + The order is summary, groups, fact index, then fact pages. Every descriptor + carries a safe relative ``path`` and a hex ``sha256``; malformed manifests + raise ``ValueError``. + """ + + partitions = manifest.get("partitions") + if not isinstance(partitions, dict): + raise ValueError("frontend bundle manifest has no partitions") + descriptors = [ + _partition_descriptor(partitions.get(key), f"partitions.{key}") + for key in ("summary", "groups", "fact_index") + ] + facts = partitions.get("facts") + if not isinstance(facts, list): + raise ValueError("frontend bundle manifest has no fact partitions") + for index, value in enumerate(facts): + descriptor = _partition_descriptor(value, f"partitions.facts[{index}]") + if descriptor.get("page") != index + 1: + raise ValueError("frontend bundle fact partition sequence is incomplete") + if not isinstance(descriptor.get("count"), int) or descriptor["count"] < 0: + raise ValueError(f"frontend bundle fact partition {index + 1} has no count") + descriptors.append(descriptor) + if len(facts) != manifest.get("page_count"): + raise ValueError("frontend bundle fact partition sequence is incomplete") + if sum(descriptor["count"] for descriptor in facts) != manifest.get("fact_count"): + raise ValueError("frontend bundle fact partition counts do not reconcile") + return descriptors + + +def verify_frontend_bundle(bundle_path: str | Path) -> dict[str, Any]: + """Verify a published frontend bundle directory and return its manifest. + + Checks the manifest schema, recomputes the SHA-256 of every partition the + manifest lists, and confirms each partition carries the manifest's schema, + run ID, and snapshot ID. Raises ``ValueError`` on any mismatch. + """ + + bundle = Path(bundle_path) + manifest_path = bundle / "manifest.json" + if not manifest_path.is_file(): + raise ValueError(f"frontend bundle has no manifest.json: {bundle}") + try: + manifest = json.loads(manifest_path.read_bytes()) + except json.JSONDecodeError as error: + raise ValueError("frontend bundle manifest.json is not valid JSON") from error + if not isinstance(manifest, dict): + raise ValueError("frontend bundle manifest.json is not an object") + if manifest.get("schema_version") != FRONTEND_BUNDLE_SCHEMA: + raise ValueError( + f"unsupported frontend bundle schema: {manifest.get('schema_version')!r}" + ) + for field in ("run_id", "snapshot_id"): + if not isinstance(manifest.get(field), str) or not manifest[field]: + raise ValueError(f"frontend bundle manifest has no {field}") + jurisdictions = manifest.get("jurisdictions") + if jurisdictions is not None and ( + not isinstance(jurisdictions, list) + or any(not isinstance(value, str) or not value for value in jurisdictions) + ): + raise ValueError("frontend bundle manifest jurisdictions must be strings") + for descriptor in frontend_bundle_partitions(manifest): + path = bundle / descriptor["path"] + if not path.is_file(): + raise ValueError(f"frontend bundle is missing {descriptor['path']}") + content = path.read_bytes() + if _sha256(content) != descriptor["sha256"]: + raise ValueError(f"frontend bundle hash mismatch for {descriptor['path']}") + try: + document = json.loads(content) + except json.JSONDecodeError as error: + raise ValueError( + f"frontend bundle partition {descriptor['path']} is not valid JSON" + ) from error + if not isinstance(document, dict) or any( + document.get(field) != manifest[field] + for field in ("schema_version", "run_id", "snapshot_id") + ): + raise ValueError( + f"frontend bundle partition {descriptor['path']} belongs to " + "another run or snapshot" + ) + return manifest diff --git a/tests/test_frontend_bundle.py b/tests/test_frontend_bundle.py index 57fc00a..3f219a3 100644 --- a/tests/test_frontend_bundle.py +++ b/tests/test_frontend_bundle.py @@ -24,7 +24,9 @@ FRONTEND_BUNDLE_SCHEMA, _build_groups, _performance_buckets, + frontend_bundle_partitions, publish_frontend_bundle, + verify_frontend_bundle, ) from evaluation_harness.full_run import build_run_summary, build_scored_results from evaluation_harness.publisher import publish_run @@ -512,3 +514,52 @@ def test_frontend_bundle_verifies_source_hashes_and_is_immutable(tmp_path: Path) publish_frontend_bundle(snapshot, run, output) with pytest.raises(FileExistsError): publish_frontend_bundle(snapshot, run, output) + + +def test_verify_frontend_bundle_accepts_published_output(tmp_path: Path) -> None: + snapshot, run = published_inputs(tmp_path) + output = tmp_path / "frontend" + manifest = publish_frontend_bundle(snapshot, run, output, page_size=2) + + verified = verify_frontend_bundle(output) + + assert verified == manifest + assert [part["path"] for part in frontend_bundle_partitions(verified)] == [ + "summary.json", + "groups.json", + "fact-index.json", + "facts/00001.json", + "facts/00002.json", + ] + + +def test_verify_frontend_bundle_rejects_tampering(tmp_path: Path) -> None: + snapshot, run = published_inputs(tmp_path) + output = tmp_path / "frontend" + publish_frontend_bundle(snapshot, run, output, page_size=2) + page = output / "facts" / "00002.json" + original = page.read_bytes() + + page.write_bytes(b'{"rows": []}\n') + with pytest.raises(ValueError, match="hash mismatch for facts/00002.json"): + verify_frontend_bundle(output) + + page.write_bytes(original) + manifest = json.loads((output / "manifest.json").read_text()) + manifest["run_id"] = "evaluation-other" + (output / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="belongs to another run or snapshot"): + verify_frontend_bundle(output) + + manifest["partitions"]["facts"].pop() + (output / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="sequence is incomplete"): + verify_frontend_bundle(output) + + manifest["schema_version"] = "cross_dataset.frontend_bundle.v0" + (output / "manifest.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError, match="unsupported frontend bundle schema"): + verify_frontend_bundle(output) + + with pytest.raises(ValueError, match="no manifest.json"): + verify_frontend_bundle(tmp_path / "missing") From 31fb1f0dc338d6ba63e865b2cb9123484f9eaf79 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 20:44:04 +0200 Subject: [PATCH 02/11] Add the publish extra with huggingface_hub Optional dependency group for uploading evaluation bundles to the Hugging Face dataset; the default install and test suite do not need it. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 1 + uv.lock | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a570cfd..9c496fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ microcosm = [ "tables>=3.10,<4", ] taxcalc-cps = ["taxcalc==6.7.1"] +publish = ["huggingface-hub>=0.27,<2"] [project.scripts] evaluation-harness = "evaluation_harness.cli:main" diff --git a/uv.lock b/uv.lock index 7930086..2169326 100644 --- a/uv.lock +++ b/uv.lock @@ -121,6 +121,9 @@ microcosm = [ { name = "policyengine-us" }, { name = "tables" }, ] +publish = [ + { name = "huggingface-hub" }, +] taxcalc-cps = [ { name = "taxcalc" }, ] @@ -132,6 +135,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "huggingface-hub", marker = "extra == 'publish'", specifier = ">=0.27,<2" }, { name = "numpy", specifier = ">=1.26,<3" }, { name = "pandas", marker = "extra == 'microcosm'", specifier = ">=2.2,<3" }, { name = "policyengine-core", marker = "extra == 'microcosm'", specifier = "==3.26.11" }, @@ -141,7 +145,7 @@ requires-dist = [ { name = "tables", marker = "extra == 'microcosm'", specifier = ">=3.10,<4" }, { name = "taxcalc", marker = "extra == 'taxcalc-cps'", specifier = "==6.7.1" }, ] -provides-extras = ["microcosm", "taxcalc-cps"] +provides-extras = ["microcosm", "taxcalc-cps", "publish"] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=8.3,<9" }] From 4851f12c173e82d13e2e3602c2a3ff58796cd2e5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 20:44:13 +0200 Subject: [PATCH 03/11] Add scripts/publish_evaluation_bundle_to_hf.py One reproducible command publishes a verified Cross-dataset frontend bundle to the policyengine/microcosm-evaluation dataset in the layout of the live Belgium bundle: //frontend/... plus /latest.json. The script verifies the bundle locally (schema, jurisdiction, partition hashes), compares it with the Hub by size and git blob id or LFS SHA-256 so a re-run is a no-op, refuses to overwrite a differing file under an existing run id, uploads the missing files and the latest.json pointer in one commit, then re-downloads manifest.json and summary.json through the public resolve URL and prints the CROSS_DATASET_ARTIFACT_BASE_URL variable the dashboard needs. --dry-run does the verification and prints the plan without network calls; the token comes only from HF_TOKEN or HUGGINGFACE_TOKEN. huggingface_hub is imported lazily so the script and its tests (fake Hub client, no network) run without the publish extra. Co-Authored-By: Claude Fable 5 --- scripts/publish_evaluation_bundle_to_hf.py | 767 +++++++++++++++++++++ tests/test_publish_evaluation_bundle.py | 736 ++++++++++++++++++++ 2 files changed, 1503 insertions(+) create mode 100644 scripts/publish_evaluation_bundle_to_hf.py create mode 100644 tests/test_publish_evaluation_bundle.py diff --git a/scripts/publish_evaluation_bundle_to_hf.py b/scripts/publish_evaluation_bundle_to_hf.py new file mode 100644 index 0000000..2d378fb --- /dev/null +++ b/scripts/publish_evaluation_bundle_to_hf.py @@ -0,0 +1,767 @@ +"""Publish a verified Cross-dataset frontend bundle to the Hugging Face dataset. + +The reference layout is the Belgium bundle on +``https://huggingface.co/datasets/policyengine/microcosm-evaluation``:: + + //frontend/{manifest,summary,groups,fact-index}.json + //frontend/facts/NNNNN.json + /latest.json + +```` is the dashboard country code for the bundle's jurisdiction. Run +directories are immutable: a re-run of the same bundle is a no-op, and a +differing file under an existing run ID is refused. ``latest.json`` is the only +mutable pointer. The Hugging Face client is imported lazily inside the upload +path so that the script is importable without the ``publish`` extra. + +Usage:: + + uv run --extra publish python scripts/publish_evaluation_bundle_to_hf.py \\ + --bundle /frontend --jurisdiction US \\ + [--repo policyengine/microcosm-evaluation] [--dry-run] [--no-latest] +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol, TextIO + +from evaluation_harness.frontend_bundle import ( + FRONTEND_BUNDLE_SCHEMA, + frontend_bundle_partitions, + verify_frontend_bundle, +) + +DEFAULT_REPO = "policyengine/microcosm-evaluation" +REPO_TYPE = "dataset" +REVISION = "main" +HUB_URL = "https://huggingface.co" +LATEST_SCHEMA_VERSION = 1 +TOKEN_ENV_VARS = ("HF_TOKEN", "HUGGINGFACE_TOKEN") +BASE_URL_ENV = "CROSS_DATASET_ARTIFACT_BASE_URL" +# Manifest jurisdictions whose dashboard country code is not their lower-case +# form. The dashboard registers Great Britain bundles under ``uk`` and accepts +# ``GB`` as an alias (frontend/lib/cross-dataset/source.ts). +COUNTRY_CODE_ALIASES = {"GB": "uk"} +VERIFY_PARTITIONS = ("manifest.json", "summary.json") +_JURISDICTION = re.compile(r"^[A-Z]{2}$") +_RUN_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_REPO_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$") + + +class PublishError(Exception): + """A refusal or verification failure; the message is shown to the operator.""" + + +@dataclass(frozen=True) +class BundleFile: + """One local file attested by the bundle manifest.""" + + relative_path: str + local_path: Path + size: int + sha256: str + git_blob_id: str + + +@dataclass(frozen=True) +class LocalBundle: + path: Path + manifest: dict[str, Any] + files: tuple[BundleFile, ...] + ignored: tuple[str, ...] + + @property + def run_id(self) -> str: + return str(self.manifest["run_id"]) + + @property + def snapshot_id(self) -> str: + return str(self.manifest["snapshot_id"]) + + @property + def jurisdictions(self) -> list[str]: + return list(self.manifest.get("jurisdictions") or []) + + +@dataclass(frozen=True) +class RemoteFile: + """A file already present in the dataset repository.""" + + path: str + size: int | None + blob_id: str | None = None + sha256: str | None = None + + +@dataclass(frozen=True) +class Upload: + path_in_repo: str + source: Path | bytes + size: int + sha256: str + + +@dataclass(frozen=True) +class PublishPlan: + repo: str + jurisdiction: str + country_code: str + run_id: str + snapshot_id: str + base_path: str + uploads: tuple[Upload, ...] + unchanged: tuple[str, ...] + latest_action: str # "write", "unchanged", or "skipped" + previous_latest: dict[str, Any] | None + extra_remote: tuple[str, ...] + remote_checked: bool + + @property + def base_url(self) -> str: + return resolve_base_url(self.repo, self.base_path) + + @property + def env_var(self) -> str: + return base_url_env_name(self.country_code) + + +class HubClient(Protocol): + """The subset of the Hub the publisher needs; faked in tests.""" + + def head(self) -> str | None: ... + + def list_files(self, prefix: str, *, recursive: bool) -> list[RemoteFile]: ... + + def commit( + self, + uploads: Sequence[Upload], + *, + message: str, + description: str | None = None, + parent_commit: str | None = None, + ) -> str | None: ... + + +Fetch = Callable[[str], bytes] + + +def _sha256(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + + +def git_blob_id(content: bytes) -> str: + """Return the git blob SHA-1 the Hub reports for a non-LFS file.""" + + return hashlib.sha1(b"blob %d\0" % len(content) + content).hexdigest() + + +def hub_token_from_env(environ: Mapping[str, str] = os.environ) -> str | None: + for name in TOKEN_ENV_VARS: + value = environ.get(name, "").strip() + if value: + return value + return None + + +def dashboard_country_code(jurisdiction: str) -> str: + """Return the dashboard country code that serves a manifest jurisdiction.""" + + return COUNTRY_CODE_ALIASES.get(jurisdiction, jurisdiction.lower()) + + +def base_url_env_name(country_code: str) -> str: + """Return the dashboard variable for a country's remote bundle (#167).""" + + if country_code == "us": + return BASE_URL_ENV + return f"{BASE_URL_ENV}_{country_code.upper()}" + + +def resolve_base_url(repo: str, base_path: str) -> str: + return f"{HUB_URL}/datasets/{repo}/resolve/{REVISION}/{base_path}" + + +def latest_document( + jurisdiction: str, run_id: str, snapshot_id: str, base_path: str +) -> bytes: + """Render ``/latest.json`` byte-for-byte like the published Belgium file.""" + + document = { + "schema_version": LATEST_SCHEMA_VERSION, + "jurisdiction": jurisdiction, + "run_id": run_id, + "snapshot_id": snapshot_id, + "base_path": base_path, + } + return json.dumps(document, indent=1).encode() + + +def normalize_jurisdiction(value: str) -> str: + jurisdiction = value.strip().upper() + if not _JURISDICTION.match(jurisdiction): + raise PublishError( + f"jurisdiction must be a two-letter code such as US or BE, not {value!r}" + ) + return jurisdiction + + +def load_bundle(bundle_path: str | Path, jurisdiction: str) -> LocalBundle: + """Verify a bundle directory and hash every file the manifest attests.""" + + bundle = Path(bundle_path) + if not bundle.is_dir(): + raise PublishError(f"bundle directory does not exist: {bundle}") + try: + manifest = verify_frontend_bundle(bundle) + except ValueError as error: + raise PublishError(f"bundle verification failed: {error}") from error + jurisdictions = manifest.get("jurisdictions") or [] + if jurisdiction not in jurisdictions: + listed = ", ".join(jurisdictions) or "(none)" + raise PublishError( + f"bundle {manifest['run_id']} is for jurisdictions {listed}, " + f"not {jurisdiction}" + ) + if not _RUN_ID.match(manifest["run_id"]): + raise PublishError(f"run_id is not a safe path segment: {manifest['run_id']!r}") + files: list[BundleFile] = [] + for relative_path in ( + "manifest.json", + *(descriptor["path"] for descriptor in frontend_bundle_partitions(manifest)), + ): + local_path = bundle / relative_path + content = local_path.read_bytes() + files.append( + BundleFile( + relative_path=relative_path, + local_path=local_path, + size=len(content), + sha256=_sha256(content), + git_blob_id=git_blob_id(content), + ) + ) + attested = {item.relative_path for item in files} + ignored = tuple( + sorted( + path.relative_to(bundle).as_posix() + for path in bundle.rglob("*") + if path.is_file() and path.relative_to(bundle).as_posix() not in attested + ) + ) + return LocalBundle( + path=bundle, manifest=manifest, files=tuple(files), ignored=ignored + ) + + +def _remote_matches( + remote: RemoteFile, *, size: int, sha256: str, git_blob_id: str +) -> bool: + """Identical on the Hub: same size and the same LFS SHA-256 or git blob id.""" + + if remote.size is not None and remote.size != size: + return False + if remote.sha256 is not None: + return remote.sha256 == sha256 + if remote.blob_id is not None: + return remote.blob_id == git_blob_id + return False + + +def _remote_latest(client: HubClient, country_code: str) -> RemoteFile | None: + path = f"{country_code}/latest.json" + for entry in client.list_files(country_code, recursive=False): + if entry.path == path: + return entry + return None + + +def build_plan( + bundle: LocalBundle, + jurisdiction: str, + *, + repo: str, + client: HubClient | None, + write_latest: bool, + fetch: Fetch | None = None, +) -> PublishPlan: + """Compare the bundle with the repository and decide what to upload. + + Without a client (dry run) every attested file is planned for upload and + the remote state is reported as unchecked. + """ + + country_code = dashboard_country_code(jurisdiction) + base_path = f"{country_code}/{bundle.run_id}/frontend/" + latest_path = f"{country_code}/latest.json" + latest_content = latest_document( + jurisdiction, bundle.run_id, bundle.snapshot_id, base_path + ) + latest_upload = Upload( + path_in_repo=latest_path, + source=latest_content, + size=len(latest_content), + sha256=_sha256(latest_content), + ) + if client is None: + uploads = [ + Upload( + path_in_repo=base_path + item.relative_path, + source=item.local_path, + size=item.size, + sha256=item.sha256, + ) + for item in bundle.files + ] + if write_latest: + uploads.append(latest_upload) + return PublishPlan( + repo=repo, + jurisdiction=jurisdiction, + country_code=country_code, + run_id=bundle.run_id, + snapshot_id=bundle.snapshot_id, + base_path=base_path, + uploads=tuple(uploads), + unchanged=(), + latest_action="write" if write_latest else "skipped", + previous_latest=None, + extra_remote=(), + remote_checked=False, + ) + + remote_by_path = { + entry.path: entry + for entry in client.list_files(base_path.rstrip("/"), recursive=True) + } + uploads = [] + unchanged = [] + conflicts = [] + for item in bundle.files: + path_in_repo = base_path + item.relative_path + remote = remote_by_path.pop(path_in_repo, None) + if remote is None: + uploads.append( + Upload( + path_in_repo=path_in_repo, + source=item.local_path, + size=item.size, + sha256=item.sha256, + ) + ) + elif _remote_matches( + remote, size=item.size, sha256=item.sha256, git_blob_id=item.git_blob_id + ): + unchanged.append(path_in_repo) + else: + conflicts.append(path_in_repo) + if conflicts: + raise PublishError( + f"run {bundle.run_id} is already published under {base_path} with " + "different content; run directories are immutable. Differing files: " + + ", ".join(conflicts) + ) + extra_remote = tuple(sorted(remote_by_path)) + + latest_action = "skipped" + previous_latest: dict[str, Any] | None = None + if write_latest: + remote_latest = _remote_latest(client, country_code) + if remote_latest is None: + latest_action = "write" + elif _remote_matches( + remote_latest, + size=latest_upload.size, + sha256=latest_upload.sha256, + git_blob_id=git_blob_id(latest_content), + ): + latest_action = "unchanged" + else: + latest_action = "write" + if fetch is not None: + previous_latest = _previous_latest(fetch, repo, latest_path) + if latest_action == "write": + uploads.append(latest_upload) + return PublishPlan( + repo=repo, + jurisdiction=jurisdiction, + country_code=country_code, + run_id=bundle.run_id, + snapshot_id=bundle.snapshot_id, + base_path=base_path, + uploads=tuple(uploads), + unchanged=tuple(unchanged), + latest_action=latest_action, + previous_latest=previous_latest, + extra_remote=extra_remote, + remote_checked=True, + ) + + +def _previous_latest( + fetch: Fetch, repo: str, latest_path: str +) -> dict[str, Any] | None: + try: + document = json.loads(fetch(resolve_base_url(repo, latest_path))) + except (OSError, ValueError): + return None + return document if isinstance(document, dict) else None + + +def commit_message(plan: PublishPlan) -> tuple[str, str]: + bundle_uploads = [ + upload + for upload in plan.uploads + if upload.path_in_repo.startswith(plan.base_path) + ] + if bundle_uploads: + message = f"Publish {plan.country_code} evaluation bundle {plan.run_id}" + else: + message = f"Point {plan.country_code}/latest.json at {plan.run_id}" + description = "\n".join( + [ + f"Jurisdiction: {plan.jurisdiction}", + f"Run: {plan.run_id}", + f"Snapshot: {plan.snapshot_id}", + f"Files uploaded: {len(bundle_uploads)}; unchanged: {len(plan.unchanged)}", + f"latest.json: {plan.latest_action}", + "Published by scripts/publish_evaluation_bundle_to_hf.py", + ] + ) + return message, description + + +def verify_published( + plan: PublishPlan, + bundle: LocalBundle, + fetch: Fetch, + *, + attempts: int = 5, + delay: float = 2.0, + sleep: Callable[[float], None] = time.sleep, +) -> list[str]: + """Re-download the key partitions through the resolve URL and check hashes.""" + + expected = {item.relative_path: item.sha256 for item in bundle.files} + checks = [ + (plan.base_url + name, expected[name], name) for name in VERIFY_PARTITIONS + ] + if plan.latest_action == "write": + content = latest_document( + plan.jurisdiction, plan.run_id, plan.snapshot_id, plan.base_path + ) + latest_path = f"{plan.country_code}/latest.json" + checks.append( + (resolve_base_url(plan.repo, latest_path), _sha256(content), latest_path) + ) + verified: list[str] = [] + for url, sha256, label in checks: + last_error = "" + for attempt in range(1, attempts + 1): + try: + actual = _sha256(fetch(url)) + except OSError as error: + last_error = str(error) + else: + if actual == sha256: + verified.append(label) + break + last_error = f"SHA-256 {actual} does not match local {sha256}" + if attempt < attempts: + sleep(delay * attempt) + else: + raise PublishError( + f"published file {url} failed verification after {attempts} " + f"attempts: {last_error}" + ) + return verified + + +def http_get(url: str, *, timeout: float = 60.0) -> bytes: + """Fetch a public resolve URL without credentials, like the dashboard does.""" + + request = urllib.request.Request( + url, headers={"User-Agent": "calibration-diagnostics-publish/1"} + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + except urllib.error.HTTPError as error: + raise OSError(f"{url} returned HTTP {error.code}") from error + except urllib.error.URLError as error: + raise OSError(f"{url} is unreachable: {error.reason}") from error + + +class HfHubClient: + """Thin adapter over ``huggingface_hub.HfApi``, imported only when used.""" + + def __init__(self, repo: str, *, token: str | None = None, api: Any = None) -> None: + if api is None: + try: + from huggingface_hub import HfApi # optional dependency: publish extra + except ImportError as error: + raise PublishError( + "huggingface_hub is not installed; run with " + "`uv run --extra publish python scripts/...` (the publish extra)" + ) from error + api = HfApi(token=token) + self._api = api + self._repo = repo + + def head(self) -> str | None: + info = self._api.repo_info(self._repo, repo_type=REPO_TYPE, revision=REVISION) + sha = getattr(info, "sha", None) + return str(sha) if sha else None + + def list_files(self, prefix: str, *, recursive: bool) -> list[RemoteFile]: + try: + entries = list( + self._api.list_repo_tree( + self._repo, + prefix, + recursive=recursive, + repo_type=REPO_TYPE, + revision=REVISION, + ) + ) + except Exception as error: + if _is_entry_not_found(error): + return [] # the prefix does not exist yet + raise + files: list[RemoteFile] = [] + for entry in entries: + if not hasattr(entry, "blob_id"): + continue # folders carry a tree id, not a blob id + lfs = getattr(entry, "lfs", None) + if isinstance(lfs, Mapping): + lfs_sha256 = lfs.get("sha256") + else: + lfs_sha256 = getattr(lfs, "sha256", None) + files.append( + RemoteFile( + path=str(entry.path), + size=getattr(entry, "size", None), + blob_id=getattr(entry, "blob_id", None), + sha256=str(lfs_sha256) if lfs_sha256 else None, + ) + ) + return files + + def commit( + self, + uploads: Sequence[Upload], + *, + message: str, + description: str | None = None, + parent_commit: str | None = None, + ) -> str | None: + from huggingface_hub import CommitOperationAdd # optional dependency + + operations = [ + CommitOperationAdd( + path_in_repo=upload.path_in_repo, path_or_fileobj=upload.source + ) + for upload in uploads + ] + info = self._api.create_commit( + repo_id=self._repo, + repo_type=REPO_TYPE, + revision=REVISION, + operations=operations, + commit_message=message, + commit_description=description, + parent_commit=parent_commit, + ) + url = getattr(info, "commit_url", None) or getattr(info, "oid", None) + return str(url) if url else None + + +def _is_entry_not_found(error: Exception) -> bool: + try: + from huggingface_hub.errors import EntryNotFoundError + except ImportError: # pragma: no cover - exercised only with a fake API + return type(error).__name__ in { + "EntryNotFoundError", + "RemoteEntryNotFoundError", + } + return isinstance(error, EntryNotFoundError) + + +def _format_size(size: int) -> str: + return f"{size:,} bytes" + + +def render_plan(plan: PublishPlan, bundle: LocalBundle, *, dry_run: bool) -> str: + lines = [ + f"Bundle: {bundle.path}", + f" schema: {FRONTEND_BUNDLE_SCHEMA}", + f" run_id: {bundle.run_id}", + f" snapshot_id: {bundle.snapshot_id}", + f" jurisdictions: {', '.join(bundle.jurisdictions)}", + f" files: {len(bundle.files)} verified (SHA-256), " + f"{_format_size(sum(item.size for item in bundle.files))}", + ] + for path in bundle.ignored: + lines.append(f" ignored: {path} (not listed in manifest.json)") + lines.append(f"Target: {plan.repo} ({REPO_TYPE}, revision {REVISION})") + lines.append(f" base path: {plan.base_path}") + if dry_run: + lines.append("Upload plan (dry run; remote state not checked):") + else: + lines.append("Upload plan:") + latest_path = f"{plan.country_code}/latest.json" + for upload in plan.uploads: + verb = "write " if upload.path_in_repo == latest_path else "upload" + lines.append(f" {verb} {upload.path_in_repo} {_format_size(upload.size)}") + for path in plan.unchanged: + lines.append(f" skip {path} (identical on the Hub)") + if plan.latest_action == "unchanged": + lines.append(f" skip {latest_path} (already points at {plan.run_id})") + elif plan.latest_action == "skipped": + lines.append(f" skip {latest_path} (--no-latest)") + if plan.previous_latest is not None: + lines.append( + f" note {latest_path} currently points at " + f"{plan.previous_latest.get('run_id')}" + ) + for path in plan.extra_remote: + lines.append(f" note {path} exists on the Hub but is not in this bundle") + return "\n".join(lines) + + +def render_result(plan: PublishPlan) -> str: + return "\n".join( + [ + "Dashboard configuration:", + f" {plan.env_var}={plan.base_url}", + ] + ) + + +def publish( + bundle_path: str | Path, + jurisdiction: str, + *, + repo: str = DEFAULT_REPO, + dry_run: bool = False, + write_latest: bool = True, + client: HubClient | None = None, + fetch: Fetch = http_get, + environ: Mapping[str, str] = os.environ, + out: TextIO | None = None, + sleep: Callable[[float], None] = time.sleep, +) -> PublishPlan: + """Verify, upload, point ``latest.json``, and re-verify one bundle. + + ``client`` and ``fetch`` are injectable so tests run without a network. + """ + + out = sys.stdout if out is None else out + if not _REPO_ID.match(repo): + raise PublishError(f"repo must be an owner/name dataset ID, not {repo!r}") + jurisdiction = normalize_jurisdiction(jurisdiction) + bundle = load_bundle(bundle_path, jurisdiction) + if dry_run: + plan = build_plan( + bundle, jurisdiction, repo=repo, client=None, write_latest=write_latest + ) + print(render_plan(plan, bundle, dry_run=True), file=out) + print(render_result(plan), file=out) + return plan + + if client is None: + token = hub_token_from_env(environ) + if token is None: + raise PublishError( + "set HF_TOKEN or HUGGINGFACE_TOKEN to a Hugging Face token with write " + f"access to {repo} (or use --dry-run)" + ) + client = HfHubClient(repo, token=token) + head = client.head() + plan = build_plan( + bundle, + jurisdiction, + repo=repo, + client=client, + write_latest=write_latest, + fetch=fetch, + ) + print(render_plan(plan, bundle, dry_run=False), file=out) + if plan.uploads: + message, description = commit_message(plan) + commit = client.commit( + plan.uploads, message=message, description=description, parent_commit=head + ) + print(f"Committed {len(plan.uploads)} file(s): {commit or message}", file=out) + else: + print("Nothing to upload: the run is already published.", file=out) + verified = verify_published(plan, bundle, fetch, sleep=sleep) + print(f"Verified through the resolve URL: {', '.join(verified)}", file=out) + print(render_result(plan), file=out) + return plan + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Publish a verified cross_dataset.frontend_bundle.v1 directory to the " + "Hugging Face evaluation dataset." + ) + ) + parser.add_argument( + "--bundle", + required=True, + type=Path, + help="frontend bundle directory (the one containing manifest.json)", + ) + parser.add_argument( + "--jurisdiction", + required=True, + help="two-letter jurisdiction the bundle must declare, such as US or BE", + ) + parser.add_argument( + "--repo", + default=DEFAULT_REPO, + help=f"dataset repository (default {DEFAULT_REPO})", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="verify the bundle and print the upload plan without network calls", + ) + parser.add_argument( + "--no-latest", + action="store_true", + help="do not point /latest.json at this run", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = _parser().parse_args(argv) + try: + publish( + arguments.bundle, + arguments.jurisdiction, + repo=arguments.repo, + dry_run=arguments.dry_run, + write_latest=not arguments.no_latest, + ) + except PublishError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_publish_evaluation_bundle.py b/tests/test_publish_evaluation_bundle.py new file mode 100644 index 0000000..98219d5 --- /dev/null +++ b/tests/test_publish_evaluation_bundle.py @@ -0,0 +1,736 @@ +import hashlib +import io +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +from evaluation_harness.frontend_bundle import FRONTEND_BUNDLE_SCHEMA +from scripts.publish_evaluation_bundle_to_hf import ( + BASE_URL_ENV, + DEFAULT_REPO, + HUB_URL, + HfHubClient, + PublishError, + RemoteFile, + Upload, + _remote_matches, + base_url_env_name, + dashboard_country_code, + git_blob_id, + hub_token_from_env, + latest_document, + load_bundle, + main, + publish, +) + +RUN_ID = "evaluation-0123456789abcdef01234567" +SNAPSHOT_ID = "chronicle-0123456789abcdef01234567" +OLD_RUN_ID = "evaluation-fedcba9876543210fedcba98" + + +def _document(value: dict[str, Any]) -> bytes: + return (json.dumps(value, sort_keys=True, separators=(",", ":")) + "\n").encode() + + +def write_bundle( + root: Path, + *, + run_id: str = RUN_ID, + snapshot_id: str = SNAPSHOT_ID, + jurisdictions: tuple[str, ...] = ("BE",), + page_counts: tuple[int, ...] = (2, 1), +) -> Path: + """Write a minimal but fully valid cross_dataset.frontend_bundle.v1 directory.""" + + common = { + "schema_version": FRONTEND_BUNDLE_SCHEMA, + "run_id": run_id, + "snapshot_id": snapshot_id, + "jurisdictions": list(jurisdictions), + } + + def write(relative: str, document: dict[str, Any]) -> dict[str, str]: + content = _document(document) + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + return {"path": relative, "sha256": hashlib.sha256(content).hexdigest()} + + fact_count = sum(page_counts) + summary = write( + "summary.json", + {**common, "fact_count": fact_count, "matrix_complete": True, "sources": []}, + ) + groups = write("groups.json", {**common, "groups": []}) + index = write("fact-index.json", {**common, "facts": {}, "facets": {}}) + facts = [] + for page, count in enumerate(page_counts, start=1): + partition = write( + f"facts/{page:05d}.json", + { + **common, + "page": page, + "page_size": max(page_counts), + "total": fact_count, + "rows": [{"fact_key": f"fact-{page}-{row}"} for row in range(count)], + }, + ) + facts.append({"page": page, "count": count, **partition}) + manifest = { + **common, + "fact_count": fact_count, + "source_ids": [], + "page_size": max(page_counts), + "page_count": len(page_counts), + "partitions": { + "summary": summary, + "groups": groups, + "fact_index": index, + "facts": facts, + }, + } + (root / "manifest.json").write_bytes(_document(manifest)) + return root + + +def bundle_paths(bundle: Path, prefix: str) -> dict[str, bytes]: + manifest = json.loads((bundle / "manifest.json").read_bytes()) + relative = ["manifest.json", "summary.json", "groups.json", "fact-index.json"] + relative.extend(part["path"] for part in manifest["partitions"]["facts"]) + return {prefix + name: (bundle / name).read_bytes() for name in relative} + + +class FakeHub: + """In-memory stand-in for the Hub: blob ids and resolve URLs, no network.""" + + def __init__( + self, files: dict[str, bytes] | None = None, repo: str = DEFAULT_REPO + ) -> None: + self.files = dict(files or {}) + self.repo = repo + self.commits: list[dict[str, Any]] = [] + + def head(self) -> str: + return f"head-{len(self.commits)}" + + def list_files(self, prefix: str, *, recursive: bool) -> list[RemoteFile]: + base = prefix.rstrip("/") + "/" + listed = [] + for path, content in sorted(self.files.items()): + if not path.startswith(base): + continue + if not recursive and "/" in path[len(base) :]: + continue + listed.append( + RemoteFile(path=path, size=len(content), blob_id=git_blob_id(content)) + ) + return listed + + def commit( + self, + uploads: list[Upload], + *, + message: str, + description: str | None = None, + parent_commit: str | None = None, + ) -> str: + for upload in uploads: + source = upload.source + content = source if isinstance(source, bytes) else Path(source).read_bytes() + self.files[upload.path_in_repo] = content + self.commits.append( + { + "message": message, + "description": description, + "parent_commit": parent_commit, + "paths": [upload.path_in_repo for upload in uploads], + } + ) + return f"{HUB_URL}/datasets/{self.repo}/commit/{len(self.commits)}" + + def fetch(self, url: str) -> bytes: + prefix = f"{HUB_URL}/datasets/{self.repo}/resolve/main/" + if not url.startswith(prefix) or url[len(prefix) :] not in self.files: + raise OSError(f"{url} returned HTTP 404") + return self.files[url[len(prefix) :]] + + +def run_publish(bundle: Path, hub: FakeHub, jurisdiction: str = "BE", **kwargs: Any): + out = io.StringIO() + plan = publish( + bundle, + jurisdiction, + client=hub, + fetch=hub.fetch, + out=out, + sleep=lambda _seconds: None, + **kwargs, + ) + return plan, out.getvalue() + + +def test_dry_run_verifies_and_prints_the_plan_without_a_client(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + out = io.StringIO() + + plan = publish(bundle, "be", dry_run=True, out=out, environ={}) + + text = out.getvalue() + prefix = f"be/{RUN_ID}/frontend/" + assert plan.remote_checked is False + assert plan.base_path == prefix + assert [upload.path_in_repo for upload in plan.uploads] == [ + *(prefix + name for name in ("manifest.json", "summary.json", "groups.json")), + prefix + "fact-index.json", + prefix + "facts/00001.json", + prefix + "facts/00002.json", + "be/latest.json", + ] + assert "dry run; remote state not checked" in text + assert f"upload {prefix}facts/00002.json" in text + assert "write be/latest.json" in text + assert ( + f"CROSS_DATASET_ARTIFACT_BASE_URL_BE={HUB_URL}/datasets/{DEFAULT_REPO}" + f"/resolve/main/{prefix}" + ) in text + + +def test_bundle_hash_mismatch_is_refused(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + (bundle / "facts" / "00002.json").write_bytes(b'{"tampered": true}\n') + + with pytest.raises(PublishError, match="hash mismatch for facts/00002.json"): + publish(bundle, "BE", dry_run=True) + + +def test_unsupported_schema_is_refused(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + manifest = json.loads((bundle / "manifest.json").read_bytes()) + manifest["schema_version"] = "cross_dataset.frontend_bundle.v2" + (bundle / "manifest.json").write_bytes(_document(manifest)) + + with pytest.raises(PublishError, match="unsupported frontend bundle schema"): + publish(bundle, "BE", dry_run=True) + + +def test_partition_from_another_run_is_refused(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + other = write_bundle(tmp_path / "other", run_id=OLD_RUN_ID) + manifest = json.loads((bundle / "manifest.json").read_bytes()) + content = (other / "groups.json").read_bytes() + (bundle / "groups.json").write_bytes(content) + manifest["partitions"]["groups"]["sha256"] = hashlib.sha256(content).hexdigest() + (bundle / "manifest.json").write_bytes(_document(manifest)) + + with pytest.raises(PublishError, match="belongs to another run"): + publish(bundle, "BE", dry_run=True) + + +def test_jurisdiction_mismatch_is_refused(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend", jurisdictions=("BE",)) + + with pytest.raises(PublishError, match="for jurisdictions BE, not US"): + publish(bundle, "US", dry_run=True) + with pytest.raises(PublishError, match="two-letter code"): + publish(bundle, "Belgium", dry_run=True) + + +def test_files_outside_the_manifest_are_ignored_not_uploaded(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + (bundle / ".DS_Store").write_bytes(b"junk") + (bundle / "facts" / "notes.txt").write_text("scratch") + + loaded = load_bundle(bundle, "BE") + hub = FakeHub() + _, text = run_publish(bundle, hub) + + assert loaded.ignored == (".DS_Store", "facts/notes.txt") + assert "ignored: .DS_Store (not listed in manifest.json)" in text + assert not any(path.endswith((".DS_Store", "notes.txt")) for path in hub.files) + + +def test_publish_uploads_the_run_and_points_latest_at_it(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + hub = FakeHub() + + plan, text = run_publish(bundle, hub) + + prefix = f"be/{RUN_ID}/frontend/" + expected = bundle_paths(bundle, prefix) + assert {path: hub.files[path] for path in expected} == expected + assert json.loads(hub.files["be/latest.json"]) == { + "schema_version": 1, + "jurisdiction": "BE", + "run_id": RUN_ID, + "snapshot_id": SNAPSHOT_ID, + "base_path": prefix, + } + assert hub.files["be/latest.json"] == latest_document( + "BE", RUN_ID, SNAPSHOT_ID, prefix + ) + assert len(hub.commits) == 1 + commit = hub.commits[0] + assert commit["message"] == f"Publish be evaluation bundle {RUN_ID}" + assert commit["parent_commit"] == "head-0" + assert commit["paths"] == [*expected, "be/latest.json"] + assert plan.latest_action == "write" + assert ( + "Verified through the resolve URL: manifest.json, summary.json, be/latest.json" + in text + ) + assert ( + f"CROSS_DATASET_ARTIFACT_BASE_URL_BE={HUB_URL}/datasets/{DEFAULT_REPO}" + f"/resolve/main/{prefix}" + ) in text + + +def test_rerun_of_a_published_bundle_is_a_no_op(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + hub = FakeHub() + run_publish(bundle, hub) + before = dict(hub.files) + + plan, text = run_publish(bundle, hub) + + assert hub.files == before + assert len(hub.commits) == 1 + assert plan.uploads == () + assert len(plan.unchanged) == 6 + assert plan.latest_action == "unchanged" + assert "Nothing to upload: the run is already published." in text + assert f"skip be/{RUN_ID}/frontend/summary.json (identical on the Hub)" in text + assert f"skip be/latest.json (already points at {RUN_ID})" in text + assert "Verified through the resolve URL: manifest.json, summary.json" in text + + +def test_partial_upload_is_completed_without_touching_existing_files( + tmp_path: Path, +) -> None: + bundle = write_bundle(tmp_path / "frontend") + prefix = f"be/{RUN_ID}/frontend/" + everything = bundle_paths(bundle, prefix) + partial = { + path: content + for path, content in everything.items() + if not path.endswith("facts/00002.json") + } + hub = FakeHub(partial) + + plan, _ = run_publish(bundle, hub) + + assert [upload.path_in_repo for upload in plan.uploads] == [ + prefix + "facts/00002.json", + "be/latest.json", + ] + assert hub.commits[0]["paths"] == [prefix + "facts/00002.json", "be/latest.json"] + assert {path: hub.files[path] for path in everything} == everything + + +def test_differing_file_under_an_existing_run_is_refused(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + prefix = f"be/{RUN_ID}/frontend/" + published = bundle_paths(bundle, prefix) + published[prefix + "summary.json"] = b'{"different": true}\n' + hub = FakeHub(published) + + with pytest.raises(PublishError, match="immutable") as error: + run_publish(bundle, hub) + + assert prefix + "summary.json" in str(error.value) + assert hub.commits == [] + assert "be/latest.json" not in hub.files + + +def test_same_size_different_content_is_still_a_conflict(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + prefix = f"be/{RUN_ID}/frontend/" + published = bundle_paths(bundle, prefix) + original = published[prefix + "groups.json"] + published[prefix + "groups.json"] = original[:-2] + b"]\n" + assert len(published[prefix + "groups.json"]) == len(original) + hub = FakeHub(published) + + with pytest.raises(PublishError, match="groups.json"): + run_publish(bundle, hub) + + +def test_no_latest_leaves_the_pointer_alone(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + old_pointer = latest_document( + "BE", OLD_RUN_ID, SNAPSHOT_ID, f"be/{OLD_RUN_ID}/frontend/" + ) + hub = FakeHub({"be/latest.json": old_pointer}) + + plan, text = run_publish(bundle, hub, write_latest=False) + + assert plan.latest_action == "skipped" + assert hub.files["be/latest.json"] == old_pointer + assert "be/latest.json" not in hub.commits[0]["paths"] + assert "skip be/latest.json (--no-latest)" in text + assert "Verified through the resolve URL: manifest.json, summary.json\n" in text + + +def test_latest_pointer_moves_from_the_previous_run(tmp_path: Path) -> None: + old_bundle = write_bundle(tmp_path / "old", run_id=OLD_RUN_ID) + hub = FakeHub() + run_publish(old_bundle, hub) + bundle = write_bundle(tmp_path / "frontend") + + plan, text = run_publish(bundle, hub) + + assert plan.latest_action == "write" + assert plan.previous_latest["run_id"] == OLD_RUN_ID + assert f"note be/latest.json currently points at {OLD_RUN_ID}" in text + assert json.loads(hub.files["be/latest.json"])["run_id"] == RUN_ID + # The previous run directory is untouched. + old_prefix = f"be/{OLD_RUN_ID}/frontend/" + assert {path: hub.files[path] for path in bundle_paths(old_bundle, old_prefix)} == ( + bundle_paths(old_bundle, old_prefix) + ) + + +def test_latest_only_commit_when_the_run_is_already_present(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + hub = FakeHub(bundle_paths(bundle, f"be/{RUN_ID}/frontend/")) + + plan, _ = run_publish(bundle, hub) + + assert [upload.path_in_repo for upload in plan.uploads] == ["be/latest.json"] + assert hub.commits[0]["message"] == f"Point be/latest.json at {RUN_ID}" + + +def test_extra_remote_files_under_the_run_are_reported(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + prefix = f"be/{RUN_ID}/frontend/" + hub = FakeHub( + {**bundle_paths(bundle, prefix), prefix + "facts/00003.json": b"{}\n"} + ) + + plan, text = run_publish(bundle, hub) + + assert plan.extra_remote == (prefix + "facts/00003.json",) + assert ( + f"note {prefix}facts/00003.json exists on the Hub but is not in this bundle" + in text + ) + + +def test_post_upload_verification_failure_is_an_error(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + hub = FakeHub() + attempts: list[str] = [] + + def stale_fetch(url: str) -> bytes: + attempts.append(url) + if url.endswith("summary.json"): + return b"stale cdn copy" + return hub.fetch(url) + + with pytest.raises(PublishError, match="summary.json failed verification after 5"): + publish( + bundle, + "BE", + client=hub, + fetch=stale_fetch, + out=io.StringIO(), + sleep=lambda _seconds: None, + ) + + assert len(hub.commits) == 1 + assert sum(url.endswith("summary.json") for url in attempts) == 5 + + +def test_publish_verifies_latest_and_retries_propagation(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + hub = FakeHub() + misses = {"count": 0} + waits: list[float] = [] + + def eventually_fetch(url: str) -> bytes: + if url.endswith("manifest.json") and misses["count"] < 2: + misses["count"] += 1 + raise OSError(f"{url} returned HTTP 404") + return hub.fetch(url) + + out = io.StringIO() + publish( + bundle, "BE", client=hub, fetch=eventually_fetch, out=out, sleep=waits.append + ) + + assert waits == [2.0, 4.0] + assert ( + "Verified through the resolve URL: manifest.json, summary.json, be/latest.json" + in (out.getvalue()) + ) + + +@pytest.mark.parametrize( + ("jurisdiction", "country_code", "env_var"), + [ + ("US", "us", BASE_URL_ENV), + ("BE", "be", f"{BASE_URL_ENV}_BE"), + ("UK", "uk", f"{BASE_URL_ENV}_UK"), + ("GB", "uk", f"{BASE_URL_ENV}_UK"), + ], +) +def test_dashboard_variable_naming_matches_the_frontend( + tmp_path: Path, jurisdiction: str, country_code: str, env_var: str +) -> None: + assert dashboard_country_code(jurisdiction) == country_code + assert base_url_env_name(country_code) == env_var + bundle = write_bundle(tmp_path / "frontend", jurisdictions=(jurisdiction,)) + out = io.StringIO() + + plan = publish(bundle, jurisdiction.lower(), dry_run=True, out=out) + + base_url = ( + f"{HUB_URL}/datasets/{DEFAULT_REPO}/resolve/main/" + f"{country_code}/{RUN_ID}/frontend/" + ) + assert plan.country_code == country_code + assert plan.env_var == env_var + assert plan.base_url == base_url + assert f" {env_var}={base_url}" in out.getvalue() + assert json.loads(plan.uploads[-1].source)["jurisdiction"] == jurisdiction + + +def test_latest_document_reproduces_the_published_belgium_pointer() -> None: + # Captured from be/latest.json on the live dataset (blob b409101a...). + content = latest_document( + "BE", + "evaluation-f28ca06a0b0d2baf13c87f2f", + "chronicle-82b574e3a8526ce0718ad08d", + "be/evaluation-f28ca06a0b0d2baf13c87f2f/frontend/", + ) + + assert content == ( + b'{\n "schema_version": 1,\n "jurisdiction": "BE",\n' + b' "run_id": "evaluation-f28ca06a0b0d2baf13c87f2f",\n' + b' "snapshot_id": "chronicle-82b574e3a8526ce0718ad08d",\n' + b' "base_path": "be/evaluation-f28ca06a0b0d2baf13c87f2f/frontend/"\n}' + ) + assert len(content) == 217 + assert git_blob_id(content) == "b409101a699bdb50f57501072fa2e5b62f7e1932" + + +def test_remote_identity_uses_size_with_lfs_sha256_or_git_blob_id() -> None: + content = b'{"a":1}\n' + size = len(content) + sha256 = hashlib.sha256(content).hexdigest() + blob_id = git_blob_id(content) + + def matches(remote: RemoteFile) -> bool: + return _remote_matches(remote, size=size, sha256=sha256, git_blob_id=blob_id) + + assert matches(RemoteFile("x", size, blob_id=blob_id)) + assert matches(RemoteFile("x", size, sha256=sha256)) + assert matches(RemoteFile("x", None, sha256=sha256)) + assert not matches(RemoteFile("x", size + 1, blob_id=blob_id)) + assert not matches(RemoteFile("x", size, blob_id="0" * 40)) + assert not matches(RemoteFile("x", size, blob_id=blob_id, sha256="0" * 64)) + assert not matches(RemoteFile("x", size)) + + +def test_token_comes_only_from_the_environment(tmp_path: Path) -> None: + assert hub_token_from_env({}) is None + assert hub_token_from_env({"HF_TOKEN": " hf_abc "}) == "hf_abc" + assert hub_token_from_env({"HUGGINGFACE_TOKEN": "hf_def"}) == "hf_def" + assert ( + hub_token_from_env({"HF_TOKEN": "", "HUGGINGFACE_TOKEN": "hf_def"}) == "hf_def" + ) + bundle = write_bundle(tmp_path / "frontend") + + with pytest.raises(PublishError, match="HF_TOKEN or HUGGINGFACE_TOKEN"): + publish(bundle, "BE", environ={}, out=io.StringIO()) + + +def test_repo_id_is_validated(tmp_path: Path) -> None: + bundle = write_bundle(tmp_path / "frontend") + + with pytest.raises(PublishError, match="owner/name"): + publish(bundle, "BE", repo="not-a-repo", dry_run=True) + + +def test_cli_exit_codes(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + bundle = write_bundle(tmp_path / "frontend") + + assert main(["--bundle", str(bundle), "--jurisdiction", "be", "--dry-run"]) == 0 + assert "be/latest.json" in capsys.readouterr().out + assert ( + main( + [ + "--bundle", + str(bundle), + "--jurisdiction", + "US", + "--dry-run", + "--no-latest", + ] + ) + == 1 + ) + captured = capsys.readouterr() + assert captured.err.startswith("error: bundle ") + assert "not US" in captured.err + + +def test_cli_no_latest_drops_the_pointer_from_the_plan( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + bundle = write_bundle(tmp_path / "frontend") + + assert ( + main( + [ + "--bundle", + str(bundle), + "--jurisdiction", + "BE", + "--dry-run", + "--no-latest", + ] + ) + == 0 + ) + text = capsys.readouterr().out + assert "write be/latest.json" not in text + assert "skip be/latest.json (--no-latest)" in text + + +@dataclass +class FakeRepoFile: + path: str + size: int + blob_id: str + lfs: dict[str, Any] | None = None + + +@dataclass +class FakeRepoFolder: + path: str + tree_id: str + + +class FakeHfApi: + """Mimics the HfApi calls the adapter makes; installs nothing, sends nothing.""" + + def __init__(self, entries: dict[str, list[Any]]) -> None: + self.entries = entries + self.calls: list[tuple[str, dict[str, Any]]] = [] + + def repo_info(self, repo_id: str, **kwargs: Any) -> Any: + self.calls.append(("repo_info", {"repo_id": repo_id, **kwargs})) + return type("Info", (), {"sha": "abc123"})() + + def list_repo_tree( + self, repo_id: str, path_in_repo: str, **kwargs: Any + ) -> list[Any]: + from huggingface_hub.errors import EntryNotFoundError + + self.calls.append( + ("list_repo_tree", {"repo_id": repo_id, "path": path_in_repo, **kwargs}) + ) + if path_in_repo not in self.entries: + raise EntryNotFoundError("Entry Not Found") + return self.entries[path_in_repo] + + def create_commit(self, **kwargs: Any) -> Any: + self.calls.append(("create_commit", kwargs)) + return type("CommitInfo", (), {"commit_url": "https://huggingface.co/c/1"})() + + +def test_hf_hub_client_adapts_hfapi_calls(tmp_path: Path) -> None: + pytest.importorskip("huggingface_hub") + from huggingface_hub import CommitOperationAdd + + api = FakeHfApi( + { + "be/run/frontend": [ + FakeRepoFolder("be/run/frontend/facts", "tree"), + FakeRepoFile("be/run/frontend/manifest.json", 10, "blob-1"), + FakeRepoFile( + "be/run/frontend/facts/00001.json", + 20, + "blob-2", + lfs={"sha256": "f" * 64, "size": 20, "pointer_size": 130}, + ), + ] + } + ) + client = HfHubClient("policyengine/microcosm-evaluation", api=api) + + assert client.head() == "abc123" + assert client.list_files("be/run/frontend", recursive=True) == [ + RemoteFile("be/run/frontend/manifest.json", 10, blob_id="blob-1"), + RemoteFile( + "be/run/frontend/facts/00001.json", 20, blob_id="blob-2", sha256="f" * 64 + ), + ] + assert client.list_files("us", recursive=False) == [] + listing = api.calls[1] + assert listing[1]["repo_type"] == "dataset" + assert listing[1]["revision"] == "main" + assert listing[1]["recursive"] is True + + page = tmp_path / "00001.json" + page.write_bytes(b"{}\n") + url = client.commit( + [ + Upload("be/run/frontend/facts/00001.json", page, 3, "0" * 64), + Upload("be/latest.json", b"{}", 2, "1" * 64), + ], + message="Publish be evaluation bundle run", + description="details", + parent_commit="abc123", + ) + + assert url == "https://huggingface.co/c/1" + commit = api.calls[-1][1] + assert commit["repo_id"] == "policyengine/microcosm-evaluation" + assert commit["repo_type"] == "dataset" + assert commit["revision"] == "main" + assert commit["commit_message"] == "Publish be evaluation bundle run" + assert commit["commit_description"] == "details" + assert commit["parent_commit"] == "abc123" + operations = commit["operations"] + assert all(isinstance(operation, CommitOperationAdd) for operation in operations) + assert [operation.path_in_repo for operation in operations] == [ + "be/run/frontend/facts/00001.json", + "be/latest.json", + ] + + +def test_script_imports_without_huggingface_hub(tmp_path: Path) -> None: + import importlib + import subprocess + import sys + + script = importlib.import_module("scripts.publish_evaluation_bundle_to_hf").__file__ + bundle = write_bundle(tmp_path / "frontend") + code = ( + "import builtins, sys\n" + "real_import = builtins.__import__\n" + "def guarded(name, *args, **kwargs):\n" + " if name.split('.')[0] == 'huggingface_hub':\n" + " raise ImportError('huggingface_hub is not installed')\n" + " return real_import(name, *args, **kwargs)\n" + "builtins.__import__ = guarded\n" + "import runpy\n" + f"sys.argv = ['publish', '--bundle', {str(bundle)!r}, " + "'--jurisdiction', 'BE', '--dry-run']\n" + f"runpy.run_path({script!r}, run_name='__main__')\n" + ) + completed = subprocess.run( + [sys.executable, "-c", code], + cwd=Path(script).resolve().parents[1], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0, completed.stderr + assert "be/latest.json" in completed.stdout From 3fda1f015d96fde434a7177ae3a37c18a50d0d18 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 20:44:13 +0200 Subject: [PATCH 04/11] Document the Hugging Face publish command Co-Authored-By: Claude Fable 5 --- docs/chronicle-update-workflow.md | 4 +- docs/cross-dataset-api.md | 63 ++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/docs/chronicle-update-workflow.md b/docs/chronicle-update-workflow.md index 64f2daa..1a7654a 100644 --- a/docs/chronicle-update-workflow.md +++ b/docs/chronicle-update-workflow.md @@ -83,7 +83,9 @@ The command refuses to use an integration reviewed against another snapshot. It classifies every fact for every registered source, runs Microcosm + PolicyEngine-US, Public CPS + Tax-Calculator, and Raw ACS PUMS, incorporates the reviewed Yale reconstruction checkpoint, scores the results, and publishes the -frontend partitions. +frontend partitions. Upload the resulting bundle to the Hugging Face dataset +with `scripts/publish_evaluation_bundle_to_hf.py`, described in +[the Cross-dataset artifact API](cross-dataset-api.md#publish-a-bundle-to-hugging-face). ## 4. Manual harness verification diff --git a/docs/cross-dataset-api.md b/docs/cross-dataset-api.md index 3e87b87..59a3637 100644 --- a/docs/cross-dataset-api.md +++ b/docs/cross-dataset-api.md @@ -53,6 +53,65 @@ run before writing. It emits a manifest, summary and group partitions, a fact index, and bounded fact pages. Every partition carries the immutable run and snapshot IDs and has a SHA-256 recorded in the manifest. +## Publish a bundle to Hugging Face + +Publish the verified bundle to the +[`policyengine/microcosm-evaluation`](https://huggingface.co/datasets/policyengine/microcosm-evaluation) +dataset with one command. It reproduces the layout of the Belgium bundle +referenced below: + +```bash +export HF_TOKEN=... # or HUGGINGFACE_TOKEN; a write token for the dataset +uv run --extra publish python scripts/publish_evaluation_bundle_to_hf.py \ + --bundle /path/to/evaluation-run/frontend \ + --jurisdiction BE +``` + +Options: `--repo ` targets another dataset, `--dry-run` verifies the +bundle and prints the upload plan without any network call (and without the +`publish` extra or a token), and `--no-latest` leaves the country's pointer +alone. The dataset layout is: + +```text +//frontend/{manifest,summary,groups,fact-index}.json +//frontend/facts/NNNNN.json +/latest.json +``` + +`` is the dashboard country code: the lower-case jurisdiction (`us`, `uk`, +`be`), with `GB`-coded bundles published under `uk`. The command: + +1. verifies the bundle locally: `schema_version` must be + `cross_dataset.frontend_bundle.v1`, the manifest's `jurisdictions` must + contain `--jurisdiction`, every listed partition must match its recorded + SHA-256, and every partition must carry the manifest's run and snapshot IDs; +2. compares the bundle with `//frontend/` on the Hub. Files that + already exist with the same size and content are skipped, so a re-run is a + no-op. A file that differs under an existing run ID aborts the publish + before anything is uploaded: run directories are immutable; +3. uploads the missing files and, unless `--no-latest`, writes + `/latest.json` in one commit: + + ```json + { + "schema_version": 1, + "jurisdiction": "BE", + "run_id": "evaluation-f28ca06a0b0d2baf13c87f2f", + "snapshot_id": "chronicle-82b574e3a8526ce0718ad08d", + "base_path": "be/evaluation-f28ca06a0b0d2baf13c87f2f/frontend/" + } + ``` + +4. re-downloads `manifest.json` and `summary.json` (and `latest.json` when + written) through the public resolve URL, exactly as the dashboard reads + them, and checks their hashes against the local bundle; +5. prints the variable to set for the dashboard, for example + `CROSS_DATASET_ARTIFACT_BASE_URL_BE=https://huggingface.co/datasets/policyengine/microcosm-evaluation/resolve/main/be//frontend/`. + +The token is read only from `HF_TOKEN` or `HUGGINGFACE_TOKEN`; the script never +reads a keychain or a cached login. Files in the bundle directory that the +manifest does not list are reported and never uploaded. + ## Configure the application Configure at most one local directory or remote base URL for each country. The @@ -89,7 +148,9 @@ export CROSS_DATASET_ARTIFACT_BASE_URL=https://example.org/evaluation-run/fronte make dev ``` -For example, the published Belgium bundle can be selected with: +A bundle published with `scripts/publish_evaluation_bundle_to_hf.py` is +selected with the base URL the command prints; `/latest.json` on the +dataset names the current run. For the published Belgium bundle: ```bash export CROSS_DATASET_ARTIFACT_BASE_URL_BE=https://huggingface.co/datasets/policyengine/microcosm-evaluation/resolve/main/be/evaluation-f28ca06a0b0d2baf13c87f2f/frontend/ From c2a49554ff3bb2d69582d85ac7034cdbb53d5888 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 21:31:54 +0200 Subject: [PATCH 05/11] Track the publish review fix pass --- PROGRESS.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 PROGRESS.md diff --git a/PROGRESS.md b/PROGRESS.md new file mode 100644 index 0000000..0c52748 --- /dev/null +++ b/PROGRESS.md @@ -0,0 +1,21 @@ +# Progress + +## State + +Fixing the five adversarial review findings for PR C on +`publish-evaluation-bundle`. + +## Done + +- Read the PR C contract in `DESIGN_BRIEF.md`. +- Confirmed the worktree is clean at `3fda1f0`. + +## Next + +- Snapshot verified bundle bytes and publish those immutable snapshots. +- Align harness partition validation with the frontend contract. +- Close and canonicalize manifest partition enumeration. +- Require remote size equality for idempotent skips. +- Cover a missing `huggingface_hub` install in a subprocess test. +- Run the complete test suite and required BE dry run. +- Write the final report to the requested output file. From 619fd416cf39a27069760d714c8e4c0dff90b6fd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 21:35:04 +0200 Subject: [PATCH 06/11] Reject malformed attested frontend partitions --- PROGRESS.md | 9 +++-- evaluation_harness/frontend_bundle.py | 45 ++++++++++++++++++++-- tests/test_frontend_bundle.py | 55 +++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 0c52748..539c6e4 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,19 +2,20 @@ ## State -Fixing the five adversarial review findings for PR C on -`publish-evaluation-bundle`. +The harness now rejects noncanonical partition manifests and fact pages whose +attested bodies disagree with their descriptors. Publisher hardening remains. ## Done - Read the PR C contract in `DESIGN_BRIEF.md`. - Confirmed the worktree is clean at `3fda1f0`. +- Rejected unknown partition keys, dot path components, and duplicate paths. +- Mirrored frontend fact-page metadata and row-count validation. +- Added correctly hashed corrupt-page and noncanonical-manifest tests. ## Next - Snapshot verified bundle bytes and publish those immutable snapshots. -- Align harness partition validation with the frontend contract. -- Close and canonicalize manifest partition enumeration. - Require remote size equality for idempotent skips. - Cover a missing `huggingface_hub` install in a subprocess test. - Run the complete test suite and required BE dry run. diff --git a/evaluation_harness/frontend_bundle.py b/evaluation_harness/frontend_bundle.py index 4d167b0..f579266 100644 --- a/evaluation_harness/frontend_bundle.py +++ b/evaluation_harness/frontend_bundle.py @@ -825,7 +825,7 @@ def _partition_descriptor(value: Any, field: str) -> dict[str, Any]: if ( path.startswith("/") or "\\" in path - or any(part in ("", "..") for part in path.split("/")) + or any(part in ("", ".", "..") for part in path.split("/")) or not _PARTITION_PATH.match(path) ): raise ValueError( @@ -847,6 +847,15 @@ def frontend_bundle_partitions(manifest: dict[str, Any]) -> list[dict[str, Any]] partitions = manifest.get("partitions") if not isinstance(partitions, dict): raise ValueError("frontend bundle manifest has no partitions") + expected_keys = {"summary", "groups", "fact_index", "facts"} + unknown_keys = set(partitions) - expected_keys + if unknown_keys: + unknown = ", ".join(sorted(str(key) for key in unknown_keys)) + raise ValueError(f"frontend bundle manifest has unknown partitions: {unknown}") + for field, minimum in (("fact_count", 0), ("page_count", 0), ("page_size", 1)): + value = manifest.get(field) + if not isinstance(value, int) or isinstance(value, bool) or value < minimum: + raise ValueError(f"frontend bundle manifest has no valid {field}") descriptors = [ _partition_descriptor(partitions.get(key), f"partitions.{key}") for key in ("summary", "groups", "fact_index") @@ -856,15 +865,20 @@ def frontend_bundle_partitions(manifest: dict[str, Any]) -> list[dict[str, Any]] raise ValueError("frontend bundle manifest has no fact partitions") for index, value in enumerate(facts): descriptor = _partition_descriptor(value, f"partitions.facts[{index}]") - if descriptor.get("page") != index + 1: + page = descriptor.get("page") + if not isinstance(page, int) or isinstance(page, bool) or page != index + 1: raise ValueError("frontend bundle fact partition sequence is incomplete") - if not isinstance(descriptor.get("count"), int) or descriptor["count"] < 0: + count = descriptor.get("count") + if not isinstance(count, int) or isinstance(count, bool) or count < 0: raise ValueError(f"frontend bundle fact partition {index + 1} has no count") descriptors.append(descriptor) if len(facts) != manifest.get("page_count"): raise ValueError("frontend bundle fact partition sequence is incomplete") if sum(descriptor["count"] for descriptor in facts) != manifest.get("fact_count"): raise ValueError("frontend bundle fact partition counts do not reconcile") + paths = [descriptor["path"] for descriptor in descriptors] + if len(paths) != len(set(paths)): + raise ValueError("frontend bundle partition paths must be unique") return descriptors @@ -899,7 +913,8 @@ def verify_frontend_bundle(bundle_path: str | Path) -> dict[str, Any]: or any(not isinstance(value, str) or not value for value in jurisdictions) ): raise ValueError("frontend bundle manifest jurisdictions must be strings") - for descriptor in frontend_bundle_partitions(manifest): + descriptors = frontend_bundle_partitions(manifest) + for index, descriptor in enumerate(descriptors): path = bundle / descriptor["path"] if not path.is_file(): raise ValueError(f"frontend bundle is missing {descriptor['path']}") @@ -920,4 +935,26 @@ def verify_frontend_bundle(bundle_path: str | Path) -> dict[str, Any]: f"frontend bundle partition {descriptor['path']} belongs to " "another run or snapshot" ) + if index >= 3: + integer_fields = { + "page": descriptor["page"], + "page_size": manifest["page_size"], + "total": manifest["fact_count"], + } + if any( + not isinstance(document.get(field), int) + or isinstance(document[field], bool) + or document[field] != expected + for field, expected in integer_fields.items() + ): + raise ValueError( + f"frontend bundle fact partition {descriptor['path']} has " + "inconsistent page metadata" + ) + rows = document.get("rows") + if not isinstance(rows, list) or len(rows) != descriptor["count"]: + raise ValueError( + f"frontend bundle fact partition {descriptor['path']} has " + "an inconsistent row count" + ) return manifest diff --git a/tests/test_frontend_bundle.py b/tests/test_frontend_bundle.py index 3f219a3..7e9670f 100644 --- a/tests/test_frontend_bundle.py +++ b/tests/test_frontend_bundle.py @@ -3,6 +3,7 @@ from dataclasses import replace from decimal import Decimal from pathlib import Path +from typing import Any import pytest @@ -563,3 +564,57 @@ def test_verify_frontend_bundle_rejects_tampering(tmp_path: Path) -> None: with pytest.raises(ValueError, match="no manifest.json"): verify_frontend_bundle(tmp_path / "missing") + + +@pytest.mark.parametrize( + ("updates", "message"), + [ + ({"page": 999}, "inconsistent page metadata"), + ({"page": True}, "inconsistent page metadata"), + ({"rows": []}, "inconsistent row count"), + ({"rows": {}}, "inconsistent row count"), + ], +) +def test_verify_frontend_bundle_rejects_attested_invalid_fact_pages( + tmp_path: Path, updates: dict[str, Any], message: str +) -> None: + snapshot, run = published_inputs(tmp_path) + output = tmp_path / "frontend" + publish_frontend_bundle(snapshot, run, output, page_size=2) + manifest_path = output / "manifest.json" + manifest = json.loads(manifest_path.read_bytes()) + page_path = output / "facts" / "00001.json" + page = json.loads(page_path.read_bytes()) + page.update(updates) + content = json.dumps(page).encode() + page_path.write_bytes(content) + manifest["partitions"]["facts"][0]["sha256"] = hashlib.sha256( + content + ).hexdigest() + manifest_path.write_text(json.dumps(manifest)) + + with pytest.raises(ValueError, match=message): + verify_frontend_bundle(output) + + +def test_frontend_bundle_partitions_rejects_noncanonical_manifests( + tmp_path: Path, +) -> None: + snapshot, run = published_inputs(tmp_path) + output = tmp_path / "frontend" + original = publish_frontend_bundle(snapshot, run, output, page_size=2) + + manifest = json.loads(json.dumps(original)) + manifest["partitions"]["extra"] = manifest["partitions"]["summary"] + with pytest.raises(ValueError, match="unknown partitions: extra"): + frontend_bundle_partitions(manifest) + + manifest = json.loads(json.dumps(original)) + manifest["partitions"]["facts"][0]["path"] = "facts/./00001.json" + with pytest.raises(ValueError, match="unsafe path"): + frontend_bundle_partitions(manifest) + + manifest = json.loads(json.dumps(original)) + manifest["partitions"]["groups"]["path"] = "summary.json" + with pytest.raises(ValueError, match="paths must be unique"): + frontend_bundle_partitions(manifest) From 5944acc62b954f1789a265a7b4b134b7e75095b9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 21:39:10 +0200 Subject: [PATCH 07/11] Publish attested bundle bytes instead of mutable paths --- PROGRESS.md | 7 ++-- scripts/publish_evaluation_bundle_to_hf.py | 49 +++++++++++++++++----- tests/test_publish_evaluation_bundle.py | 46 ++++++++++++++++---- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 539c6e4..a840767 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,8 +2,8 @@ ## State -The harness now rejects noncanonical partition manifests and fact pages whose -attested bodies disagree with their descriptors. Publisher hardening remains. +Verified bundle files are now immutable byte snapshots from loading through +commit. Remote comparison and missing-extra coverage remain. ## Done @@ -12,10 +12,11 @@ attested bodies disagree with their descriptors. Publisher hardening remains. - Rejected unknown partition keys, dot path components, and duplicate paths. - Mirrored frontend fact-page metadata and row-count validation. - Added correctly hashed corrupt-page and noncanonical-manifest tests. +- Replaced mutable upload paths with descriptor-validated byte snapshots. +- Added a pre-commit disk-mutation regression test. ## Next -- Snapshot verified bundle bytes and publish those immutable snapshots. - Require remote size equality for idempotent skips. - Cover a missing `huggingface_hub` install in a subprocess test. - Run the complete test suite and required BE dry run. diff --git a/scripts/publish_evaluation_bundle_to_hf.py b/scripts/publish_evaluation_bundle_to_hf.py index 2d378fb..d481757 100644 --- a/scripts/publish_evaluation_bundle_to_hf.py +++ b/scripts/publish_evaluation_bundle_to_hf.py @@ -68,7 +68,7 @@ class BundleFile: """One local file attested by the bundle manifest.""" relative_path: str - local_path: Path + content: bytes size: int sha256: str git_blob_id: str @@ -107,7 +107,7 @@ class RemoteFile: @dataclass(frozen=True) class Upload: path_in_repo: str - source: Path | bytes + source: bytes size: int sha256: str @@ -222,10 +222,26 @@ def load_bundle(bundle_path: str | Path, jurisdiction: str) -> LocalBundle: bundle = Path(bundle_path) if not bundle.is_dir(): raise PublishError(f"bundle directory does not exist: {bundle}") + manifest_path = bundle / "manifest.json" + try: + manifest_content = manifest_path.read_bytes() + except OSError as error: + raise PublishError( + "bundle verification failed: frontend bundle has no manifest.json: " + f"{bundle}" + ) from error try: manifest = verify_frontend_bundle(bundle) except ValueError as error: raise PublishError(f"bundle verification failed: {error}") from error + try: + manifest_changed = manifest_path.read_bytes() != manifest_content + except OSError as error: + raise PublishError( + "bundle manifest changed while it was being loaded" + ) from error + if manifest_changed: + raise PublishError("bundle manifest changed while it was being loaded") jurisdictions = manifest.get("jurisdictions") or [] if jurisdiction not in jurisdictions: listed = ", ".join(jurisdictions) or "(none)" @@ -235,19 +251,30 @@ def load_bundle(bundle_path: str | Path, jurisdiction: str) -> LocalBundle: ) if not _RUN_ID.match(manifest["run_id"]): raise PublishError(f"run_id is not a safe path segment: {manifest['run_id']!r}") - files: list[BundleFile] = [] - for relative_path in ( - "manifest.json", - *(descriptor["path"] for descriptor in frontend_bundle_partitions(manifest)), - ): + files = [ + BundleFile( + relative_path="manifest.json", + content=manifest_content, + size=len(manifest_content), + sha256=_sha256(manifest_content), + git_blob_id=git_blob_id(manifest_content), + ) + ] + for descriptor in frontend_bundle_partitions(manifest): + relative_path = descriptor["path"] local_path = bundle / relative_path content = local_path.read_bytes() + sha256 = _sha256(content) + if sha256 != descriptor["sha256"]: + raise PublishError( + f"bundle partition {relative_path} changed while it was being loaded" + ) files.append( BundleFile( relative_path=relative_path, - local_path=local_path, + content=content, size=len(content), - sha256=_sha256(content), + sha256=sha256, git_blob_id=git_blob_id(content), ) ) @@ -317,7 +344,7 @@ def build_plan( uploads = [ Upload( path_in_repo=base_path + item.relative_path, - source=item.local_path, + source=item.content, size=item.size, sha256=item.sha256, ) @@ -354,7 +381,7 @@ def build_plan( uploads.append( Upload( path_in_repo=path_in_repo, - source=item.local_path, + source=item.content, size=item.size, sha256=item.sha256, ) diff --git a/tests/test_publish_evaluation_bundle.py b/tests/test_publish_evaluation_bundle.py index 98219d5..57b1b15 100644 --- a/tests/test_publish_evaluation_bundle.py +++ b/tests/test_publish_evaluation_bundle.py @@ -139,9 +139,7 @@ def commit( parent_commit: str | None = None, ) -> str: for upload in uploads: - source = upload.source - content = source if isinstance(source, bytes) else Path(source).read_bytes() - self.files[upload.path_in_repo] = content + self.files[upload.path_in_repo] = upload.source self.commits.append( { "message": message, @@ -288,6 +286,41 @@ def test_publish_uploads_the_run_and_points_latest_at_it(tmp_path: Path) -> None ) in text +def test_publish_uses_attested_bytes_if_a_file_changes_before_commit( + tmp_path: Path, +) -> None: + bundle = write_bundle(tmp_path / "frontend") + page = bundle / "facts" / "00002.json" + attested = page.read_bytes() + tampered = b'{"tampered":true}\n' + + class MutatingHub(FakeHub): + def commit( + self, + uploads: list[Upload], + *, + message: str, + description: str | None = None, + parent_commit: str | None = None, + ) -> str: + page.write_bytes(tampered) + return super().commit( + uploads, + message=message, + description=description, + parent_commit=parent_commit, + ) + + hub = MutatingHub() + + run_publish(bundle, hub) + + published = hub.files[f"be/{RUN_ID}/frontend/facts/00002.json"] + assert page.read_bytes() == tampered + assert published == attested + assert published != tampered + + def test_rerun_of_a_published_bundle_is_a_no_op(tmp_path: Path) -> None: bundle = write_bundle(tmp_path / "frontend") hub = FakeHub() @@ -643,7 +676,7 @@ def create_commit(self, **kwargs: Any) -> Any: return type("CommitInfo", (), {"commit_url": "https://huggingface.co/c/1"})() -def test_hf_hub_client_adapts_hfapi_calls(tmp_path: Path) -> None: +def test_hf_hub_client_adapts_hfapi_calls() -> None: pytest.importorskip("huggingface_hub") from huggingface_hub import CommitOperationAdd @@ -676,11 +709,9 @@ def test_hf_hub_client_adapts_hfapi_calls(tmp_path: Path) -> None: assert listing[1]["revision"] == "main" assert listing[1]["recursive"] is True - page = tmp_path / "00001.json" - page.write_bytes(b"{}\n") url = client.commit( [ - Upload("be/run/frontend/facts/00001.json", page, 3, "0" * 64), + Upload("be/run/frontend/facts/00001.json", b"{}\n", 3, "0" * 64), Upload("be/latest.json", b"{}", 2, "1" * 64), ], message="Publish be evaluation bundle run", @@ -702,6 +733,7 @@ def test_hf_hub_client_adapts_hfapi_calls(tmp_path: Path) -> None: "be/run/frontend/facts/00001.json", "be/latest.json", ] + assert [operation.path_or_fileobj for operation in operations] == [b"{}\n", b"{}"] def test_script_imports_without_huggingface_hub(tmp_path: Path) -> None: From fd89fe982d36969dac066a86a71cd0c3a53250a0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 21:39:29 +0200 Subject: [PATCH 08/11] Require remote size before skipping bundle files --- PROGRESS.md | 6 +++--- scripts/publish_evaluation_bundle_to_hf.py | 2 +- tests/test_publish_evaluation_bundle.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index a840767..2a117e9 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,8 +2,8 @@ ## State -Verified bundle files are now immutable byte snapshots from loading through -commit. Remote comparison and missing-extra coverage remain. +Verified bundle files are immutable byte snapshots, and remote skips now +require both size and hash identity. Missing-extra coverage remains. ## Done @@ -14,10 +14,10 @@ commit. Remote comparison and missing-extra coverage remain. - Added correctly hashed corrupt-page and noncanonical-manifest tests. - Replaced mutable upload paths with descriptor-validated byte snapshots. - Added a pre-commit disk-mutation regression test. +- Required a present, equal remote size before either hash form can skip. ## Next -- Require remote size equality for idempotent skips. - Cover a missing `huggingface_hub` install in a subprocess test. - Run the complete test suite and required BE dry run. - Write the final report to the requested output file. diff --git a/scripts/publish_evaluation_bundle_to_hf.py b/scripts/publish_evaluation_bundle_to_hf.py index d481757..43cfa17 100644 --- a/scripts/publish_evaluation_bundle_to_hf.py +++ b/scripts/publish_evaluation_bundle_to_hf.py @@ -296,7 +296,7 @@ def _remote_matches( ) -> bool: """Identical on the Hub: same size and the same LFS SHA-256 or git blob id.""" - if remote.size is not None and remote.size != size: + if remote.size is None or remote.size != size: return False if remote.sha256 is not None: return remote.sha256 == sha256 diff --git a/tests/test_publish_evaluation_bundle.py b/tests/test_publish_evaluation_bundle.py index 57b1b15..fb84829 100644 --- a/tests/test_publish_evaluation_bundle.py +++ b/tests/test_publish_evaluation_bundle.py @@ -561,7 +561,8 @@ def matches(remote: RemoteFile) -> bool: assert matches(RemoteFile("x", size, blob_id=blob_id)) assert matches(RemoteFile("x", size, sha256=sha256)) - assert matches(RemoteFile("x", None, sha256=sha256)) + assert not matches(RemoteFile("x", None, blob_id=blob_id)) + assert not matches(RemoteFile("x", None, sha256=sha256)) assert not matches(RemoteFile("x", size + 1, blob_id=blob_id)) assert not matches(RemoteFile("x", size, blob_id="0" * 40)) assert not matches(RemoteFile("x", size, blob_id=blob_id, sha256="0" * 64)) From 0db37718d68b4cf16a184b240eaca527a178521b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 21:40:19 +0200 Subject: [PATCH 09/11] Test missing publish extra before network access --- PROGRESS.md | 6 +-- tests/test_publish_evaluation_bundle.py | 52 +++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 2a117e9..90045d1 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,8 +2,8 @@ ## State -Verified bundle files are immutable byte snapshots, and remote skips now -require both size and hash identity. Missing-extra coverage remains. +All five review findings are implemented with focused regression coverage. +The required full-suite and BE dry-run gates remain. ## Done @@ -15,9 +15,9 @@ require both size and hash identity. Missing-extra coverage remains. - Replaced mutable upload paths with descriptor-validated byte snapshots. - Added a pre-commit disk-mutation regression test. - Required a present, equal remote size before either hash form can skip. +- Covered the missing publish extra in a non-dry subprocess with network guards. ## Next -- Cover a missing `huggingface_hub` install in a subprocess test. - Run the complete test suite and required BE dry run. - Write the final report to the requested output file. diff --git a/tests/test_publish_evaluation_bundle.py b/tests/test_publish_evaluation_bundle.py index fb84829..34f762c 100644 --- a/tests/test_publish_evaluation_bundle.py +++ b/tests/test_publish_evaluation_bundle.py @@ -767,3 +767,55 @@ def test_script_imports_without_huggingface_hub(tmp_path: Path) -> None: assert completed.returncode == 0, completed.stderr assert "be/latest.json" in completed.stdout + + +def test_non_dry_cli_fails_without_huggingface_hub_before_network( + tmp_path: Path, +) -> None: + import importlib + import os + import subprocess + import sys + + script = importlib.import_module("scripts.publish_evaluation_bundle_to_hf").__file__ + bundle = write_bundle(tmp_path / "frontend") + network_marker = tmp_path / "network-attempted" + code = ( + "import builtins, pathlib, socket, sys\n" + f"marker = pathlib.Path({str(network_marker)!r})\n" + "def network_attempt(*args, **kwargs):\n" + " marker.write_text('attempted')\n" + " raise AssertionError('network attempted')\n" + "class GuardedSocket(socket.socket):\n" + " connect = network_attempt\n" + " connect_ex = network_attempt\n" + "socket.socket = GuardedSocket\n" + "socket.create_connection = network_attempt\n" + "socket.getaddrinfo = network_attempt\n" + "real_import = builtins.__import__\n" + "def guarded(name, *args, **kwargs):\n" + " if name.split('.')[0] == 'huggingface_hub':\n" + " raise ImportError('huggingface_hub is not installed')\n" + " return real_import(name, *args, **kwargs)\n" + "builtins.__import__ = guarded\n" + "import runpy\n" + f"sys.argv = ['publish', '--bundle', {str(bundle)!r}, " + "'--jurisdiction', 'BE']\n" + f"runpy.run_path({script!r}, run_name='__main__')\n" + ) + environment = os.environ.copy() + environment.pop("HUGGINGFACE_TOKEN", None) + environment["HF_TOKEN"] = "not-a-credential" + + completed = subprocess.run( + [sys.executable, "-c", code], + cwd=Path(script).resolve().parents[1], + env=environment, + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 1 + assert "uv run --extra publish" in completed.stderr + assert not network_marker.exists() From e8f1c2534c774258ba8ecfa4a8501b1246dd43f8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 21:43:17 +0200 Subject: [PATCH 10/11] Record passing gates for publish race and validation fixes --- PROGRESS.md | 10 ++++++---- tests/test_frontend_bundle.py | 4 +--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 90045d1..c504503 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -2,8 +2,7 @@ ## State -All five review findings are implemented with focused regression coverage. -The required full-suite and BE dry-run gates remain. +Complete. All five review findings are implemented, covered, and gated. ## Done @@ -16,8 +15,11 @@ The required full-suite and BE dry-run gates remain. - Added a pre-commit disk-mutation regression test. - Required a present, equal remote size before either hash form can skip. - Covered the missing publish extra in a non-dry subprocess with network guards. +- Passed `ruff check` on all changed Python files. +- Passed the full suite with `.venv/bin/python -m pytest -q`: 441 tests. +- Passed the required BE dry run: 12 files and 2,668,113 bytes verified. +- Used the permitted pytest fallback because the sandbox blocked the uv cache. ## Next -- Run the complete test suite and required BE dry run. -- Write the final report to the requested output file. +- None. diff --git a/tests/test_frontend_bundle.py b/tests/test_frontend_bundle.py index 7e9670f..2f2ae09 100644 --- a/tests/test_frontend_bundle.py +++ b/tests/test_frontend_bundle.py @@ -588,9 +588,7 @@ def test_verify_frontend_bundle_rejects_attested_invalid_fact_pages( page.update(updates) content = json.dumps(page).encode() page_path.write_bytes(content) - manifest["partitions"]["facts"][0]["sha256"] = hashlib.sha256( - content - ).hexdigest() + manifest["partitions"]["facts"][0]["sha256"] = hashlib.sha256(content).hexdigest() manifest_path.write_text(json.dumps(manifest)) with pytest.raises(ValueError, match=message): From 033439c680e30a6ccf1c201370c507f361a62af9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 23 Aug 2026 21:44:58 +0200 Subject: [PATCH 11/11] Remove the lane progress tracker Lane bookkeeping, not repository content. Co-Authored-By: Claude Fable 5 --- PROGRESS.md | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 PROGRESS.md diff --git a/PROGRESS.md b/PROGRESS.md deleted file mode 100644 index c504503..0000000 --- a/PROGRESS.md +++ /dev/null @@ -1,25 +0,0 @@ -# Progress - -## State - -Complete. All five review findings are implemented, covered, and gated. - -## Done - -- Read the PR C contract in `DESIGN_BRIEF.md`. -- Confirmed the worktree is clean at `3fda1f0`. -- Rejected unknown partition keys, dot path components, and duplicate paths. -- Mirrored frontend fact-page metadata and row-count validation. -- Added correctly hashed corrupt-page and noncanonical-manifest tests. -- Replaced mutable upload paths with descriptor-validated byte snapshots. -- Added a pre-commit disk-mutation regression test. -- Required a present, equal remote size before either hash form can skip. -- Covered the missing publish extra in a non-dry subprocess with network guards. -- Passed `ruff check` on all changed Python files. -- Passed the full suite with `.venv/bin/python -m pytest -q`: 441 tests. -- Passed the required BE dry run: 12 files and 2,668,113 bytes verified. -- Used the permitted pytest fallback because the sandbox blocked the uv cache. - -## Next - -- None.