From 8f358a706aec06eadf8b2bcdc9a2e7e1f3f8100c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Juaristi?= <127882282+juaristi22@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:52:26 +0200 Subject: [PATCH] Publish per-package consumer artifacts --- README.md | 29 ++++ chronicle/artifacts.py | 213 +++++++++++++++++++++++ chronicle/cli.py | 1 + chronicle/harness.py | 105 ++++++++++- docs/agent-source-package-harness.md | 20 +++ docs/storage-architecture.md | 12 ++ policyengine_chronicle/__init__.py | 2 + policyengine_chronicle/cli.py | 3 +- policyengine_chronicle/consumer.py | 250 ++++++++++++++++++++++++++- tests/test_chronicle_artifacts.py | 131 ++++++++++++++ tests/test_chronicle_consumer.py | 106 ++++++++++++ wrangler.toml | 3 +- 12 files changed, 863 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 576919cd..41a5608e 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,35 @@ only `consumer_facts.jsonl` and `manifest.json`. Version 2 is incompatible with retired v1 profile-bearing contract: loaders reject v1 manifests so downstreams must adopt the facts-only surface explicitly. +To build and publish a hash-pinned artifact for one public source package, use +the package path directly: + +```bash +uv run chronicle build-consumer-artifact \ + --package hmrc-tax-free-childcare-march-2026 \ + --year 2026 \ + --out /tmp/hmrc-tfc-consumer-artifact +uv run chronicle publish-consumer --dir /tmp/hmrc-tfc-consumer-artifact +``` + +`--package` runs the package's full build suite before producing the two-file +artifact. Its manifest records `source_id`, `package_id`, +`chronicle_source_commit`, `source_access`, `fact_row_count`, `facts_sha256`, +and `artifact_sha256`. The artifact digest is the SHA-256 of the canonical JSON +manifest fields excluding `artifact_sha256`; because those fields include +`facts_sha256`, it pins both the package identity and the exact fact bytes. + +`publish-consumer` verifies the artifact and uploads only `manifest.json` and +`consumer_facts.jsonl` to `ledger-derived` under: + +```text +consumer/{source_id}/{package_id}/{artifact_sha256}/{artifact_name} +``` + +Publication fails closed when a manifest declares `licensed` or `restricted` +access. Those sources remain hash-only registrations; only public source +packages may produce downloadable consumer artifacts. + `--year` is inert for `--suite uk` because the UK packages are year-pinned. The US off-year bundle behavior is unchanged and out of scope here. diff --git a/chronicle/artifacts.py b/chronicle/artifacts.py index d9620761..fa456944 100644 --- a/chronicle/artifacts.py +++ b/chronicle/artifacts.py @@ -23,6 +23,7 @@ DEFAULT_R2_DERIVED_BUCKET = "ledger-derived" DEFAULT_R2_PREFIX = "raw" DEFAULT_R2_DERIVED_PREFIX = "derived" +DEFAULT_R2_CONSUMER_PREFIX = "consumer" # New UK and New Zealand uploads are namespaced by country. US objects predate # the country segment and deliberately keep their legacy ``raw/{source_id}`` @@ -378,6 +379,78 @@ def to_dict(self) -> dict[str, Any]: } +@dataclass(frozen=True) +class ConsumerArtifactUploadEntry: + """One public consumer-artifact file upload status.""" + + artifact_name: str + local_path: str + sha256: str + size_bytes: int + r2_location: ArtifactStorageLocation + upload: ArtifactCommandResult + + @property + def valid(self) -> bool: + """Whether this consumer-artifact file uploaded successfully.""" + return self.upload.ok + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serializable entry.""" + return { + "valid": self.valid, + "artifact_name": self.artifact_name, + "local_path": self.local_path, + "sha256": self.sha256, + "size_bytes": self.size_bytes, + "r2_location": self.r2_location.to_dict(), + "upload": self.upload.to_dict(), + } + + +@dataclass(frozen=True) +class ConsumerArtifactPublishReport: + """Report from publishing one public, package-scoped consumer artifact.""" + + input_dir: str + source_id: str + package_id: str + artifact_sha256: str + entries: tuple[ConsumerArtifactUploadEntry, ...] + errors: tuple[str, ...] = () + + @property + def valid(self) -> bool: + """Whether both consumer-artifact files uploaded successfully.""" + return ( + not self.errors + and len(self.entries) == 2 + and all(entry.valid for entry in self.entries) + ) + + @property + def counts(self) -> dict[str, int]: + """Return summary counts.""" + return { + "artifact_count": len(self.entries), + "uploaded_count": sum(1 for entry in self.entries if entry.valid), + "failed_count": sum(1 for entry in self.entries if not entry.valid), + } + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serializable report.""" + return { + "valid": self.valid, + "input_dir": self.input_dir, + "source_id": self.source_id, + "package_id": self.package_id, + "artifact_sha256": self.artifact_sha256, + "counts": self.counts, + "entries": [entry.to_dict() for entry in self.entries], + "errors": list(self.errors), + } + + def fetch_source_artifact( source_url: str, *, @@ -590,6 +663,117 @@ def publish_derived_artifacts( return report +def publish_consumer_artifact( + input_dir: str | Path, + *, + r2_bucket: str = DEFAULT_R2_DERIVED_BUCKET, + r2_prefix: str = DEFAULT_R2_CONSUMER_PREFIX, + wrangler_command: str = "npx wrangler", +) -> ConsumerArtifactPublishReport: + """Publish a verified public package artifact at a content-addressed key.""" + from policyengine_chronicle.consumer import load_consumer_artifact + + input_path = Path(input_dir) + empty_report = { + "input_dir": str(input_path), + "source_id": "", + "package_id": "", + "artifact_sha256": "", + "entries": (), + } + manifest_path = input_path / "manifest.json" + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return ConsumerArtifactPublishReport( + **empty_report, + errors=(f"consumer_manifest_unreadable:{exc}",), + ) + + source_id = manifest.get("source_id") + package_id = manifest.get("package_id") + artifact_sha256 = manifest.get("artifact_sha256") + source_access = manifest.get("source_access") + missing = [ + field + for field, value in { + "source_id": source_id, + "package_id": package_id, + "chronicle_source_commit": manifest.get("chronicle_source_commit"), + "source_access": source_access, + "artifact_sha256": artifact_sha256, + }.items() + if not isinstance(value, str) or not value + ] + identity = { + "input_dir": str(input_path), + "source_id": str(source_id or ""), + "package_id": str(package_id or ""), + "artifact_sha256": str(artifact_sha256 or ""), + "entries": (), + } + if missing: + return ConsumerArtifactPublishReport( + **identity, + errors=("missing_manifest_fields:" + ",".join(missing),), + ) + if source_access != "public": + return ConsumerArtifactPublishReport( + **identity, + errors=(f"source_access_not_public:{source_access}",), + ) + + try: + load_consumer_artifact(input_path) + except (KeyError, OSError, TypeError, ValueError) as exc: + return ConsumerArtifactPublishReport( + **identity, + errors=(f"consumer_artifact_invalid:{exc}",), + ) + + entries: list[ConsumerArtifactUploadEntry] = [] + errors: list[str] = [] + for artifact_name in ("consumer_facts.jsonl", "manifest.json"): + artifact_path = input_path / artifact_name + content = artifact_path.read_bytes() + location = ArtifactStorageLocation( + provider="r2", + bucket=r2_bucket, + key=build_consumer_r2_key( + source_id=source_id, + package_id=package_id, + artifact_sha256=artifact_sha256, + artifact_name=artifact_name, + prefix=r2_prefix, + ), + ) + upload = _upload_r2_object( + location, + artifact_path, + wrangler_command=wrangler_command, + ) + if not upload.ok: + errors.append(f"consumer_upload_failed:{artifact_name}") + entries.append( + ConsumerArtifactUploadEntry( + artifact_name=artifact_name, + local_path=str(artifact_path), + sha256=hashlib.sha256(content).hexdigest(), + size_bytes=len(content), + r2_location=location, + upload=upload, + ) + ) + return ConsumerArtifactPublishReport( + input_dir=str(input_path), + source_id=source_id, + package_id=package_id, + artifact_sha256=artifact_sha256, + entries=tuple(entries), + errors=tuple(errors), + ) + + def publish_source_artifacts( root: str | Path, *, @@ -951,6 +1135,28 @@ def build_derived_r2_key( ) +def build_consumer_r2_key( + *, + source_id: str, + package_id: str, + artifact_sha256: str, + artifact_name: str, + prefix: str = DEFAULT_R2_CONSUMER_PREFIX, +) -> str: + """Build the canonical content-addressed key for a consumer artifact file.""" + if len(artifact_sha256) != 64 or any( + character not in "0123456789abcdef" for character in artifact_sha256 + ): + raise ValueError("Consumer artifact SHA-256 must be 64 lowercase hex digits.") + return posixpath.join( + *_clean_relative_key_parts(prefix), + _clean_consumer_key_part(source_id), + _clean_consumer_key_part(package_id), + artifact_sha256, + *_clean_relative_key_parts(artifact_name), + ) + + def build_artifact_key( *, build_id: str, @@ -1293,3 +1499,10 @@ def _clean_relative_key_parts(value: str) -> tuple[str, ...]: if not parts or any(part == ".." for part in parts): raise ValueError("R2 artifact paths cannot be empty or contain '..'.") return parts + + +def _clean_consumer_key_part(value: str) -> str: + cleaned = _clean_key_part(value) + if "/" in cleaned or cleaned in {".", ".."}: + raise ValueError("Consumer artifact source/package IDs must be one key part.") + return cleaned diff --git a/chronicle/cli.py b/chronicle/cli.py index b9098d7f..15299aae 100644 --- a/chronicle/cli.py +++ b/chronicle/cli.py @@ -24,6 +24,7 @@ def main() -> None: ["inventory-artifacts"], ["load-supabase-mirror"], ["plan-pe-sources"], + ["publish-consumer"], ["publish-derived"], ["publish-raw"], ["scaffold-package"], diff --git a/chronicle/harness.py b/chronicle/harness.py index 1d96f093..290ffbaa 100644 --- a/chronicle/harness.py +++ b/chronicle/harness.py @@ -10,12 +10,14 @@ from chronicle.artifacts import ( ArtifactFetchReport, ArtifactInventoryReport, + ConsumerArtifactPublishReport, DerivedArtifactPublishReport, R2BootstrapReport, RawArtifactPublishReport, bootstrap_r2_buckets, fetch_source_artifact, inventory_source_artifacts, + publish_consumer_artifact, publish_derived_artifacts, publish_source_artifacts, ) @@ -428,6 +430,22 @@ def publish_derived_artifact_files( ) +def publish_consumer_artifact_files( + input_dir: str | Path, + *, + r2_bucket: str = "ledger-derived", + r2_prefix: str = "consumer", + wrangler_command: str = "npx wrangler", +) -> ConsumerArtifactPublishReport: + """Publish a verified package consumer artifact to content-addressed R2.""" + return publish_consumer_artifact( + input_dir, + r2_bucket=r2_bucket, + r2_prefix=r2_prefix, + wrangler_command=wrangler_command, + ) + + def export_chronicle_db_table_files( db_path: str | Path, output_dir: str | Path, @@ -757,12 +775,34 @@ def main(argv: list[str] | None = None) -> int: help="Build a versioned facts-only consumer artifact", description="Build a versioned facts-only consumer artifact.", ) - consumer_artifact_parser.add_argument( + consumer_artifact_input = consumer_artifact_parser.add_mutually_exclusive_group( + required=True + ) + consumer_artifact_input.add_argument( "--facts", type=Path, - required=True, help="Path to a consumer_facts.jsonl file or a bundle directory", ) + consumer_artifact_input.add_argument( + "--package", + help=( + "Public source package alias, directory, or source_package.yaml to " + "build before creating the artifact" + ), + ) + consumer_artifact_parser.add_argument( + "--year", + type=int, + default=2023, + help="Source year used with --package (default: 2023)", + ) + consumer_artifact_parser.add_argument( + "--source-commit", + help=( + "Chronicle Git commit to pin with --package. Defaults to the current " + "checkout HEAD." + ), + ) consumer_artifact_parser.add_argument( "--out", type=Path, @@ -1037,6 +1077,32 @@ def main(argv: list[str] | None = None) -> int: help="Optional path to write build_artifacts JSONL rows.", ) + consumer_publish_parser = subparsers.add_parser( + "publish-consumer", + help="Upload a package consumer artifact at a content-addressed R2 key", + ) + consumer_publish_parser.add_argument( + "--dir", + type=Path, + required=True, + help="Consumer artifact directory containing manifest.json and facts.", + ) + consumer_publish_parser.add_argument( + "--r2-bucket", + default="ledger-derived", + help="R2 bucket for public consumer artifacts.", + ) + consumer_publish_parser.add_argument( + "--r2-prefix", + default="consumer", + help="Stable R2 prefix for public consumer artifacts.", + ) + consumer_publish_parser.add_argument( + "--wrangler-command", + default="npx wrangler", + help="Wrangler command prefix to use for R2 uploads.", + ) + mirror_export_parser = subparsers.add_parser( "export-db-tables", help="Export a Chronicle SQLite DB artifact to per-table JSONL files", @@ -1274,13 +1340,27 @@ def main(argv: list[str] | None = None) -> int: print(json.dumps(report.to_dict(), indent=2, sort_keys=True)) return 0 if report.valid else 1 if args.command == "build-consumer-artifact": - from policyengine_chronicle.consumer import build_consumer_artifact - - artifact_report = build_consumer_artifact( - args.out, - facts_path=args.facts, - replace=args.replace, + from policyengine_chronicle.consumer import ( + build_consumer_artifact, + build_package_consumer_artifact, ) + + if args.package: + artifact_report = build_package_consumer_artifact( + args.out, + package=args.package, + year=args.year, + chronicle_source_commit=args.source_commit, + replace=args.replace, + ) + else: + if args.source_commit: + raise ValueError("--source-commit requires --package.") + artifact_report = build_consumer_artifact( + args.out, + facts_path=args.facts, + replace=args.replace, + ) print(json.dumps(artifact_report.to_dict(), indent=2, sort_keys=True)) return 0 if args.command == "validate-package": @@ -1359,6 +1439,15 @@ def main(argv: list[str] | None = None) -> int: ) print(json.dumps(report.to_dict(), indent=2, sort_keys=True)) return 0 if report.valid else 1 + if args.command == "publish-consumer": + report = publish_consumer_artifact_files( + args.dir, + r2_bucket=args.r2_bucket, + r2_prefix=args.r2_prefix, + wrangler_command=args.wrangler_command, + ) + print(json.dumps(report.to_dict(), indent=2, sort_keys=True)) + return 0 if report.valid else 1 if args.command == "export-db-tables": report = export_chronicle_db_table_files( args.db, diff --git a/docs/agent-source-package-harness.md b/docs/agent-source-package-harness.md index 5816ec47..224bee86 100644 --- a/docs/agent-source-package-harness.md +++ b/docs/agent-source-package-harness.md @@ -579,6 +579,26 @@ uv run chronicle build-bundle --suite uk --out /tmp/chronicle-uk --replace uv run chronicle build-consumer-artifact --facts /tmp/chronicle-uk --out /tmp/chronicle-uk-artifact --replace ``` +When a downstream test needs a small package fixture instead of the merged UK +feed, build and publish the package-scoped artifact: + +```bash +uv run chronicle build-consumer-artifact \ + --package dfe-funded-early-education-childcare-2026 \ + --year 2026 \ + --out /tmp/dfe-childcare-consumer-artifact \ + --replace +uv run chronicle publish-consumer \ + --dir /tmp/dfe-childcare-consumer-artifact +``` + +The manifest records the package ID, source ID, Chronicle commit, public access +class, row count, facts hash, and artifact hash. Publication verifies all of +those fields and writes the two files to +`consumer/{source_id}/{package_id}/{artifact_sha256}/` in `ledger-derived`. +Packages declaring `licensed` or `restricted` access are rejected before any +upload. + `--year` is inert for `--suite uk` because the UK packages are year-pinned. The US off-year bundle behavior is unchanged and out of scope here. diff --git a/docs/storage-architecture.md b/docs/storage-architecture.md index cf898933..fbd5be28 100644 --- a/docs/storage-architecture.md +++ b/docs/storage-architecture.md @@ -84,6 +84,18 @@ Legacy US derived keys likewise remain `derived/{source_id}/...`. Derived artifacts are reproducible and may be replaced by a new build, but a specific `{build_id}` path should be immutable once published. +Public, package-scoped consumer artifacts use the existing `ledger-derived` +bucket and a country-neutral content-addressed key: + +```text +consumer/{source_id}/{package_id}/{artifact_sha256}/{artifact_name} +``` + +Each leaf contains exactly `manifest.json` and `consumer_facts.jsonl`. The +manifest pins the package, Chronicle source commit, row count, fact-file hash, +and canonical artifact hash. Explicitly `licensed` or `restricted` packages +are never published at this surface. + ## Relational Registry Contract The hosted `chronicle` schema should be the lookup surface for Chronicle, not the place diff --git a/policyengine_chronicle/__init__.py b/policyengine_chronicle/__init__.py index 06399e5c..44999022 100644 --- a/policyengine_chronicle/__init__.py +++ b/policyengine_chronicle/__init__.py @@ -29,6 +29,7 @@ from policyengine_chronicle.consumer import ( ConsumerArtifact, build_consumer_artifact, + build_package_consumer_artifact, load_consumer_artifact, ) @@ -51,6 +52,7 @@ "ValidationReport", "build_aggregate_constraints", "build_consumer_artifact", + "build_package_consumer_artifact", "build_fact_key", "build_label", "load_consumer_artifact", diff --git a/policyengine_chronicle/cli.py b/policyengine_chronicle/cli.py index 43a9b7fd..f92c0ff8 100644 --- a/policyengine_chronicle/cli.py +++ b/policyengine_chronicle/cli.py @@ -19,7 +19,8 @@ def main() -> None: " validate-facts\n" " validate-source-cells\n" " export-consumer-facts\n" - " build-consumer-artifact\n\n" + " build-consumer-artifact\n" + " publish-consumer\n\n" "Run `chronicle --help` for command-specific help." ) return diff --git a/policyengine_chronicle/consumer.py b/policyengine_chronicle/consumer.py index f1d808e5..5bc74857 100644 --- a/policyengine_chronicle/consumer.py +++ b/policyengine_chronicle/consumer.py @@ -10,12 +10,18 @@ import hashlib import json import math +import re import shutil +import subprocess +import tempfile from collections.abc import Mapping from dataclasses import asdict, dataclass +from importlib.resources import files from pathlib import Path from typing import Any +import yaml + from chronicle.consumer_contract import _hash_key from chronicle.core import ( ALLOWED_ASSERTIONS, @@ -28,6 +34,14 @@ ) CONSUMER_ARTIFACT_SCHEMA_VERSION = "policyengine_ledger.consumer_artifact.v2" +SOURCE_ACCESS_VALUES = frozenset({"public", "licensed", "restricted"}) +_GIT_COMMIT_RE = re.compile(r"^[0-9a-f]{40,64}$") +_PACKAGE_IDENTITY_FIELDS = ( + "source_id", + "package_id", + "chronicle_source_commit", + "source_access", +) @dataclass(frozen=True) @@ -46,10 +60,13 @@ class ConsumerArtifactBuildReport: schema_version: str output_dir: str fact_row_count: int + artifact_sha256: str + source_id: str | None = None + package_id: str | None = None def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable report.""" - return asdict(self) + return {key: value for key, value in asdict(self).items() if value is not None} def build_consumer_artifact( @@ -57,6 +74,10 @@ def build_consumer_artifact( *, facts_path: str | Path, replace: bool = False, + source_id: str | None = None, + package_id: str | None = None, + chronicle_source_commit: str | None = None, + source_access: str | None = None, ) -> ConsumerArtifactBuildReport: """Build a reproducible facts-only artifact from consumer fact rows. @@ -65,6 +86,12 @@ def build_consumer_artifact( that pins their schema and content hashes. Target contracts are packaged by the consumer, not Chronicle. """ + package_identity = _validate_package_identity( + source_id=source_id, + package_id=package_id, + chronicle_source_commit=chronicle_source_commit, + source_access=source_access, + ) output_path = Path(output_dir) if output_path.exists(): if not replace: @@ -81,6 +108,7 @@ def build_consumer_artifact( file.write(json.dumps(row, sort_keys=True)) file.write("\n") + facts_sha256 = _sha256_file(facts_out) manifest = { "schema_version": CONSUMER_ARTIFACT_SCHEMA_VERSION, "consumer_fact_schema_versions": sorted( @@ -88,16 +116,73 @@ def build_consumer_artifact( ), "consumer_fact_schema_sha256": CONSUMER_FACT_SCHEMA_SHA256, "fact_row_count": len(rows), - "facts_sha256": _sha256_file(facts_out), + "facts_sha256": facts_sha256, + **package_identity, } + artifact_sha256 = _artifact_manifest_sha256(manifest) + manifest["artifact_sha256"] = artifact_sha256 _write_json(output_path / "manifest.json", manifest) return ConsumerArtifactBuildReport( schema_version=CONSUMER_ARTIFACT_SCHEMA_VERSION, output_dir=str(output_path), fact_row_count=len(rows), + artifact_sha256=artifact_sha256, + source_id=source_id, + package_id=package_id, + ) + + +def build_package_consumer_artifact( + output_dir: str | Path, + *, + package: str | Path, + year: int = 2023, + chronicle_source_commit: str | None = None, + replace: bool = False, +) -> ConsumerArtifactBuildReport: + """Build a public package and wrap its facts as a pinned consumer artifact.""" + from chronicle.source_package import load_source_package + from chronicle.suite import build_source_suite + + output_path = Path(output_dir) + if output_path.exists() and not replace: + raise FileExistsError( + f"Output directory exists: {output_path}. Pass replace=True." + ) + + source_package = load_source_package(package) + source_id, source_access = _source_package_publication_identity( + source_package, + year=year, + ) + source_commit = _resolve_chronicle_source_commit( + chronicle_source_commit, + package_path=source_package.package_path, ) + with tempfile.TemporaryDirectory(prefix="chronicle-consumer-suite-") as tmp: + suite_dir = Path(tmp) / "suite" + suite_report = build_source_suite( + source_package.package_path, + suite_dir, + year=year, + ) + if not suite_report.valid: + raise ValueError( + "Cannot publish a consumer artifact from an invalid source-package " + f"suite: {source_package.package_id}." + ) + return build_consumer_artifact( + output_path, + facts_path=suite_dir, + replace=replace, + source_id=source_id, + package_id=source_package.package_id, + chronicle_source_commit=source_commit, + source_access=source_access, + ) + def load_consumer_artifact(path: str | Path) -> ConsumerArtifact: """Load a facts-only consumer artifact and verify its manifest hashes.""" @@ -113,6 +198,7 @@ def load_consumer_artifact(path: str | Path) -> ConsumerArtifact: "Consumer artifact manifests must not contain profiles; target profiles " "are consumer-owned contracts and must be loaded by Microcosm." ) + _validate_manifest_package_identity(manifest) manifest_schema_sha256 = manifest.get("consumer_fact_schema_sha256") if ( manifest_schema_sha256 is not None @@ -138,6 +224,14 @@ def load_consumer_artifact(path: str | Path) -> ConsumerArtifact: "Consumer artifact manifest declares fact_row_count " f"{declared_row_count} but the feed carries {len(rows)} rows." ) + artifact_sha256 = manifest.get("artifact_sha256") + recomputed_artifact_sha256 = _artifact_manifest_sha256(manifest) + if artifact_sha256 is not None and artifact_sha256 != recomputed_artifact_sha256: + raise ValueError( + "Consumer artifact manifest declares artifact_sha256 " + f"{artifact_sha256!r}, but its canonical artifact identity hashes " + f"to {recomputed_artifact_sha256!r}." + ) return ConsumerArtifact( path=artifact_path, manifest=manifest, @@ -145,6 +239,145 @@ def load_consumer_artifact(path: str | Path) -> ConsumerArtifact: ) +def _validate_package_identity( + *, + source_id: str | None, + package_id: str | None, + chronicle_source_commit: str | None, + source_access: str | None, +) -> dict[str, str]: + identity = { + "source_id": source_id, + "package_id": package_id, + "chronicle_source_commit": chronicle_source_commit, + "source_access": source_access, + } + supplied = [value is not None for value in identity.values()] + if any(supplied) and not all(supplied): + missing = [key for key, value in identity.items() if value is None] + raise ValueError( + "Package consumer-artifact identity is incomplete; missing " + + ", ".join(missing) + + "." + ) + if not any(supplied): + return {} + malformed = [key for key, value in identity.items() if not isinstance(value, str)] + if malformed: + raise ValueError( + "Package consumer-artifact identity fields must be strings: " + + ", ".join(malformed) + + "." + ) + assert source_id is not None + assert package_id is not None + assert chronicle_source_commit is not None + assert source_access is not None + if not source_id.strip() or not package_id.strip(): + raise ValueError( + "Package consumer-artifact source and package IDs are required." + ) + if not _GIT_COMMIT_RE.fullmatch(chronicle_source_commit): + raise ValueError( + "chronicle_source_commit must be a 40- to 64-character lowercase " + "hexadecimal Git object ID." + ) + if source_access not in SOURCE_ACCESS_VALUES: + raise ValueError(f"Unsupported source access class: {source_access!r}.") + if source_access != "public": + raise ValueError( + "Consumer artifacts may be built only for public source packages; " + f"{package_id!r} is {source_access!r}." + ) + return {key: value for key, value in identity.items() if isinstance(value, str)} + + +def _validate_manifest_package_identity(manifest: Mapping[str, Any]) -> None: + identity = {key: manifest.get(key) for key in _PACKAGE_IDENTITY_FIELDS} + supplied = [value is not None for value in identity.values()] + if not any(supplied): + return + if not all(supplied): + missing = [key for key, value in identity.items() if value is None] + raise ValueError( + "Consumer artifact package identity is incomplete; missing " + + ", ".join(missing) + + "." + ) + _validate_package_identity( + source_id=identity["source_id"], + package_id=identity["package_id"], + chronicle_source_commit=identity["chronicle_source_commit"], + source_access=identity["source_access"], + ) + + +def _source_package_publication_identity( + source_package, *, year: int +) -> tuple[str, str]: + manifest_path = files(source_package.artifact.resource_package).joinpath( + source_package.artifact.resource_directory, + source_package.artifact.manifest, + ) + with manifest_path.open("r", encoding="utf-8") as file: + manifest = yaml.safe_load(file) or {} + manifest_package_id = manifest.get("package_id") + if manifest_package_id and manifest_package_id != source_package.package_id: + raise ValueError( + "Source-package and artifact manifest package IDs disagree: " + f"{source_package.package_id!r} != {manifest_package_id!r}." + ) + source_id = manifest.get("source_id") or source_package.artifact.source_name + if not isinstance(source_id, str) or not source_id.strip(): + raise ValueError("Source artifact manifest needs a non-empty string source_id.") + artifact_year = source_package.artifact.artifact_year or year + files_by_year = manifest.get("files") or {} + file_spec = files_by_year.get(artifact_year) or files_by_year.get( + str(artifact_year) + ) + file_access = file_spec.get("access") if isinstance(file_spec, dict) else None + # Fact-bearing packages predate #221's explicit access field and contain + # redistributable publisher tables. Missing access therefore retains the + # legacy public classification; explicit licensed/restricted values fail. + source_access = file_access or manifest.get("access") or "public" + if not isinstance(source_access, str): + raise ValueError("Source artifact access must be a string.") + _validate_package_identity( + source_id=source_id, + package_id=source_package.package_id, + chronicle_source_commit="0" * 40, + source_access=source_access, + ) + return source_id, source_access + + +def _resolve_chronicle_source_commit( + explicit: str | None, + *, + package_path: Path, +) -> str: + if explicit is not None: + if not _GIT_COMMIT_RE.fullmatch(explicit): + raise ValueError( + "chronicle_source_commit must be a 40- to 64-character lowercase " + "hexadecimal Git object ID." + ) + return explicit + result = subprocess.run( + ["git", "-C", str(package_path), "rev-parse", "HEAD"], + check=False, + capture_output=True, + text=True, + ) + commit = result.stdout.strip().lower() + if result.returncode != 0 or not _GIT_COMMIT_RE.fullmatch(commit): + raise ValueError( + "Could not determine the Chronicle source commit. Pass " + "chronicle_source_commit explicitly." + ) + return commit + + def _resolve_facts_path(facts_path: str | Path) -> Path: path = Path(facts_path) if path.is_dir(): @@ -273,6 +506,18 @@ def _write_json(path: Path, payload: dict[str, Any]) -> None: path.write_text(json.dumps(payload, sort_keys=True, indent=2) + "\n") +def _artifact_manifest_sha256(manifest: Mapping[str, Any]) -> str: + identity = { + key: value for key, value in manifest.items() if key != "artifact_sha256" + } + canonical = json.dumps( + identity, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + def _sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as file: @@ -286,5 +531,6 @@ def _sha256_file(path: Path) -> str: "ConsumerArtifact", "ConsumerArtifactBuildReport", "build_consumer_artifact", + "build_package_consumer_artifact", "load_consumer_artifact", ] diff --git a/tests/test_chronicle_artifacts.py b/tests/test_chronicle_artifacts.py index e08beabc..30ab859a 100644 --- a/tests/test_chronicle_artifacts.py +++ b/tests/test_chronicle_artifacts.py @@ -11,6 +11,7 @@ from chronicle.cli import main as cli_main from chronicle.artifacts import ( build_artifact_rows, + build_consumer_r2_key, build_derived_r2_key, bootstrap_r2_buckets, build_r2_key, @@ -19,9 +20,11 @@ infer_build_id, inventory_source_artifacts, publish_derived_artifacts, + publish_consumer_artifact, publish_source_artifacts, ) from chronicle.harness import main as harness_main +from policyengine_chronicle.consumer import build_consumer_artifact def test_build_r2_key_is_content_addressed(): @@ -51,6 +54,32 @@ def test_build_derived_r2_key_is_build_scoped(): ) +def test_build_consumer_r2_key_is_package_and_content_addressed(): + artifact_sha256 = "ab" * 32 + + key = build_consumer_r2_key( + source_id="hmrc", + package_id="hmrc-tax-free-childcare-march-2026", + artifact_sha256=artifact_sha256, + artifact_name="manifest.json", + ) + + assert key == ( + "consumer/hmrc/hmrc-tax-free-childcare-march-2026/" + f"{artifact_sha256}/manifest.json" + ) + + +def test_build_consumer_r2_key_rejects_path_like_package_ids(): + with pytest.raises(ValueError, match="must be one key part"): + build_consumer_r2_key( + source_id="hmrc", + package_id="../restricted", + artifact_sha256="ab" * 32, + artifact_name="manifest.json", + ) + + @pytest.mark.parametrize( ("source_id", "package_path", "expected_country"), [ @@ -489,6 +518,108 @@ def test_publish_derived_artifacts_uploads_build_directory(tmp_path): assert "reports/build_summary.json" in command_log +def test_publish_consumer_artifact_uploads_only_the_pinned_pair(tmp_path): + artifact_dir = tmp_path / "artifact" + build_consumer_artifact( + artifact_dir, + facts_path="chronicle/fixtures/consumer_facts.jsonl", + source_id="irs_soi", + package_id="soi-table-1-1", + chronicle_source_commit="ab" * 20, + source_access="public", + ) + manifest = json.loads((artifact_dir / "manifest.json").read_text()) + log = tmp_path / "wrangler.log" + wrangler = tmp_path / "wrangler" + wrangler.write_text(f"#!/bin/sh\nprintf '%s\\n' \"$*\" >> {log}\necho ok\n") + wrangler.chmod(0o755) + + report = publish_consumer_artifact( + artifact_dir, + wrangler_command=str(wrangler), + ) + + assert report.valid + assert report.artifact_sha256 == manifest["artifact_sha256"] + assert report.counts == { + "artifact_count": 2, + "failed_count": 0, + "uploaded_count": 2, + } + assert {entry.artifact_name for entry in report.entries} == { + "consumer_facts.jsonl", + "manifest.json", + } + command_log = log.read_text() + stable_prefix = ( + f"ledger-derived/consumer/irs_soi/soi-table-1-1/{manifest['artifact_sha256']}" + ) + assert command_log.count(stable_prefix) == 2 + + +def test_publish_consumer_artifact_refuses_nonpublic_manifest(tmp_path): + artifact_dir = tmp_path / "artifact" + build_consumer_artifact( + artifact_dir, + facts_path="chronicle/fixtures/consumer_facts.jsonl", + source_id="irs_soi", + package_id="soi-table-1-1", + chronicle_source_commit="ab" * 20, + source_access="public", + ) + manifest_path = artifact_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["source_access"] = "restricted" + manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n") + log = tmp_path / "wrangler.log" + wrangler = tmp_path / "wrangler" + wrangler.write_text(f"#!/bin/sh\nprintf '%s\\n' \"$*\" >> {log}\necho ok\n") + wrangler.chmod(0o755) + + report = publish_consumer_artifact( + artifact_dir, + wrangler_command=str(wrangler), + ) + + assert not report.valid + assert report.errors == ("source_access_not_public:restricted",) + assert not log.exists() + + +def test_publish_consumer_cli_emits_the_stable_locations(tmp_path, capsys): + artifact_dir = tmp_path / "artifact" + build_consumer_artifact( + artifact_dir, + facts_path="chronicle/fixtures/consumer_facts.jsonl", + source_id="irs_soi", + package_id="soi-table-1-1", + chronicle_source_commit="ab" * 20, + source_access="public", + ) + wrangler = tmp_path / "wrangler" + wrangler.write_text("#!/bin/sh\necho ok\n") + wrangler.chmod(0o755) + + exit_code = harness_main( + [ + "publish-consumer", + "--dir", + str(artifact_dir), + "--wrangler-command", + str(wrangler), + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["valid"] + assert payload["counts"]["uploaded_count"] == 2 + assert all( + entry["r2_location"]["key"].startswith("consumer/irs_soi/soi-table-1-1/") + for entry in payload["entries"] + ) + + def test_build_artifact_rows_skips_failed_uploads(tmp_path): suite = tmp_path / "suite" reports = suite / "reports" diff --git a/tests/test_chronicle_consumer.py b/tests/test_chronicle_consumer.py index a7334710..6040da15 100644 --- a/tests/test_chronicle_consumer.py +++ b/tests/test_chronicle_consumer.py @@ -21,6 +21,7 @@ from chronicle.harness import main from policyengine_chronicle.consumer import ( build_consumer_artifact, + build_package_consumer_artifact, load_consumer_artifact, ) from policyengine_chronicle.schema import CONSUMER_FACT_SCHEMA_SHA256 @@ -86,9 +87,19 @@ def _rewrite_manifest_hash(out_dir): manifest_path = out_dir / "manifest.json" manifest = json.loads(manifest_path.read_text()) manifest["facts_sha256"] = hashlib.sha256(facts_file.read_bytes()).hexdigest() + manifest["artifact_sha256"] = _artifact_manifest_sha256(manifest) manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n") +def _artifact_manifest_sha256(manifest): + identity = { + key: value for key, value in manifest.items() if key != "artifact_sha256" + } + return hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + def test_artifact_build_load_round_trip_is_facts_only(tmp_path): facts_path = _write_facts(tmp_path) out_dir = tmp_path / "artifact" @@ -97,10 +108,15 @@ def test_artifact_build_load_round_trip_is_facts_only(tmp_path): artifact = load_consumer_artifact(out_dir) manifest = json.loads((out_dir / "manifest.json").read_text()) + manifest_without_artifact_sha = { + key: value for key, value in manifest.items() if key != "artifact_sha256" + } + expected_artifact_sha = _artifact_manifest_sha256(manifest_without_artifact_sha) assert report.to_dict() == { "schema_version": "policyengine_ledger.consumer_artifact.v2", "output_dir": str(out_dir), "fact_row_count": 2, + "artifact_sha256": expected_artifact_sha, } assert manifest == { "schema_version": "policyengine_ledger.consumer_artifact.v2", @@ -110,6 +126,7 @@ def test_artifact_build_load_round_trip_is_facts_only(tmp_path): "facts_sha256": hashlib.sha256( (out_dir / "consumer_facts.jsonl").read_bytes() ).hexdigest(), + "artifact_sha256": expected_artifact_sha, } assert {path.name for path in out_dir.iterdir()} == { "consumer_facts.jsonl", @@ -131,6 +148,57 @@ def test_artifact_is_reproducible(tmp_path): assert (first / name).read_bytes() == (second / name).read_bytes() +@pytest.mark.parametrize( + ("package_id", "source_id", "fact_row_count"), + [ + ("hmrc-tax-free-childcare-march-2026", "hmrc", 126), + ("dfe-funded-early-education-childcare-2026", "dfe", 770), + ], +) +def test_package_artifact_manifest_pins_package_commit_and_rows( + tmp_path, + package_id, + source_id, + fact_row_count, +): + out_dir = tmp_path / "artifact" + source_commit = "ab" * 20 + + report = build_package_consumer_artifact( + out_dir, + package=package_id, + year=2026, + chronicle_source_commit=source_commit, + ) + artifact = load_consumer_artifact(out_dir) + + assert report.fact_row_count == fact_row_count + assert report.package_id == package_id + assert artifact.manifest["source_id"] == source_id + assert artifact.manifest["package_id"] == package_id + assert artifact.manifest["chronicle_source_commit"] == source_commit + assert artifact.manifest["source_access"] == "public" + assert artifact.manifest["artifact_sha256"] == _artifact_manifest_sha256( + artifact.manifest + ) + assert artifact.manifest["fact_row_count"] == fact_row_count + + +@pytest.mark.parametrize("source_access", ["licensed", "restricted"]) +def test_artifact_build_refuses_nonpublic_package_identity(tmp_path, source_access): + facts_path = _write_facts(tmp_path) + + with pytest.raises(ValueError, match="public source packages"): + build_consumer_artifact( + tmp_path / "artifact", + facts_path=facts_path, + source_id="example", + package_id="example-package", + chronicle_source_commit="ab" * 20, + source_access=source_access, + ) + + def test_artifact_load_rejects_tampered_facts(tmp_path): facts_path = _write_facts(tmp_path) out_dir = tmp_path / "artifact" @@ -147,6 +215,19 @@ def test_artifact_load_rejects_tampered_facts(tmp_path): load_consumer_artifact(out_dir) +def test_artifact_load_rejects_false_artifact_sha256(tmp_path): + facts_path = _write_facts(tmp_path) + out_dir = tmp_path / "artifact" + build_consumer_artifact(out_dir, facts_path=facts_path) + manifest_path = out_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text()) + manifest["artifact_sha256"] = "00" * 32 + manifest_path.write_text(json.dumps(manifest, sort_keys=True, indent=2) + "\n") + + with pytest.raises(ValueError, match="artifact_sha256"): + load_consumer_artifact(out_dir) + + def test_artifact_load_rejects_profile_metadata(tmp_path): facts_path = _write_facts(tmp_path) out_dir = tmp_path / "artifact" @@ -282,3 +363,28 @@ def test_build_consumer_artifact_help_has_no_profile_options(capsys): help_text = capsys.readouterr().out assert "--profile" not in help_text assert "facts-only" in help_text + assert "--package" in help_text + + +def test_build_consumer_artifact_cli_accepts_a_package(tmp_path, capsys): + out_dir = tmp_path / "artifact" + + exit_code = main( + [ + "build-consumer-artifact", + "--package", + "hmrc-tax-free-childcare-march-2026", + "--year", + "2026", + "--source-commit", + "ab" * 20, + "--out", + str(out_dir), + ] + ) + payload = json.loads(capsys.readouterr().out) + + assert exit_code == 0 + assert payload["package_id"] == "hmrc-tax-free-childcare-march-2026" + assert payload["fact_row_count"] == 126 + assert load_consumer_artifact(out_dir).manifest["source_access"] == "public" diff --git a/wrangler.toml b/wrangler.toml index 4f8bc0e9..2f5c9857 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,5 +1,6 @@ # Pins the PolicyEngine Cloudflare account for the `wrangler r2` calls made by -# `chronicle publish-raw` / `publish-derived` / `fetch-artifact --upload-r2`. +# `chronicle publish-raw` / `publish-derived` / `publish-consumer` / +# `fetch-artifact --upload-r2`. # Without a pinned account, contributors whose Cloudflare user belongs to more # than one account fail in non-interactive runs at the account-selection step. account_id = "20d90f557651969925eece96e58e24dc"