diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c424c145d..d1845710c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -298,7 +298,7 @@ jobs: - name: Verify clean wheel boundary and import every shard run: | env -u PYTHONPATH /tmp/wheels-venv/bin/python -I - <<'PY' - from importlib import metadata, util + from importlib import metadata, resources, util from pathlib import Path import sys @@ -307,6 +307,7 @@ jobs: import microcosm.data import microcosm.fit import microcosm.frame + import microcosm.graph assert util.find_spec("policyengine_us") is None try: @@ -323,11 +324,18 @@ jobs: microcosm.calibrate, microcosm.build, microcosm.data, + microcosm.graph, ) for module in modules: path = Path(module.__file__).resolve() assert path.is_relative_to(prefix), f"source import escaped wheel venv: {path}" print(module.__name__, module.__version__, path) + + graph_schema = resources.files("microcosm.graph").joinpath("schema") + assert sorted(path.name for path in graph_schema.iterdir()) == [ + "graph-module-v1.schema.json", + "graph-source-v1.schema.json", + ] PY - name: Spec identity section digests (cross-environment diffing aid) run: env -u PYTHONPATH /tmp/wheels-venv/bin/python -I tools/spec_envelope_digests.py be uk diff --git a/changelog.d/standardize-runtime-graph.added.md b/changelog.d/standardize-runtime-graph.added.md new file mode 100644 index 000000000..7c4cd802c --- /dev/null +++ b/changelog.d/standardize-runtime-graph.added.md @@ -0,0 +1 @@ +Added authoritative Graph YAML composition, explicit population-state transitions, graph-bound run manifests, saved-run reconstruction, and post-run local materialization. diff --git a/docs/graph-acceptance.md b/docs/graph-acceptance.md index e99d36353..ba10f1e96 100644 --- a/docs/graph-acceptance.md +++ b/docs/graph-acceptance.md @@ -315,6 +315,22 @@ Amendments so far (each re-locked): records the interface amendment for owner review, not a claim of code approval or release certification. +20. **Authored runtime graph and named state semantics.** `Graph.products` + gives stable names to population states, coordinates, weight states, typed + artifacts, validation outcomes, and post-run exports. `SourceRef` declares + its content type, access classification, expected content identities, and + authoritative decoder. `Node.requires_success` expresses required + validation dependencies. `StructuralDelta.REVISION` creates a new + unchanged-row population state with declared rewrites, and + `StructuralDelta.UNION` combines two or more compatible population states + with deterministic identifiers and stored lineage. `WeightTransition.anchor` + names the earlier weight state used for ratio and mass validation. Nested + parameters are recursively immutable finite JSON values. These declaration + changes are serialized losslessly, compiled into dependency order, and + included in semantic graph and affected node identities. Adopted by the + independently owned `standardize-runtime-graph` child of PR #873 on + 2026-09-08; the country migrations remain separate changes. + Adding a normative field with a default changes the canonical projection of every node that carries it, so node keys moved with amendments 11 and 13's sibling field `entrants`; no released artifact pins a graph key yet. diff --git a/docs/graph-explorer.md b/docs/graph-explorer.md index 0f4fd9f12..2cd231775 100644 --- a/docs/graph-explorer.md +++ b/docs/graph-explorer.md @@ -26,15 +26,18 @@ Two invocations with the same code produce byte-identical HTML. ## Render any saved run -Save both sides of the review contract: +Use the authoritative YAML root and save the manifest. If the run also emits +optional Graph JSON evidence, write the versioned generated document: ```python from pathlib import Path -from microcosm.graph import graph_to_json +from microcosm.graph import graph_document_to_json manifest.save(Path("run/manifest.json")) -Path("run/graph.json").write_text(graph_to_json(graph), encoding="utf-8") +Path("run/graph.json").write_text( + graph_document_to_json(graph), encoding="utf-8" +) ``` By default, put the run's `ContentStore` at `run/store`, beside the manifest. @@ -43,12 +46,14 @@ Then render it with: ```bash uv run python tools/graph_explain.py \ --manifest run/manifest.json \ - --graph run/graph.json \ + --graph path/to/graph.yaml \ --out run/explain.html ``` If the store is elsewhere, add `--store /path/to/store`. The renderer uses -`graph_from_json`, compiles the graph again, validates the manifest and every +the authoritative YAML by default; it also accepts generated versioned Graph +JSON and older unversioned Graph JSON evidence. It compiles the graph again, +requires the saved manifest's semantic graph identity to match, validates every referenced artifact through `RunManifest.load`, and reloads structural frames from each receipt's `frame_key`. A missing or corrupt store is an error, not an unverified page. @@ -76,7 +81,7 @@ not read or write files. The large SVG is laid out from `CompiledGraph.order` and `predecessors`. Horizontal position is topological depth. Dashed background groups are the population versions from `CompiledGraph.versions`, including structural -`create`, `filter`, `expand`, and `reweight` boundaries. +`create`, `filter`, `expand`, `reweight`, `revision`, and `union` boundaries. Every node shows its id, kernel reference, role, structural delta, abbreviated node key, and store hit or miss. Blue fill means a store miss and green fill @@ -174,9 +179,11 @@ downstream node is not executed after the owning boundary rejects the dtype. ## Portable evidence boundaries Portable manifest JSON intentionally omits attached populations and transient -mass ledgers. Structural frame artifacts let the CLI recover weights, strata, -and before/after totals, but the current receipt exposes only the realized -maximum weight ratio, not the distribution's samples or bins. The immediate -post-run demo therefore carries richer attached evidence than a manifest copied -without its store. Missing evidence is identified in the page wherever it cannot -be reconstructed faithfully. +mass-ledger objects. A matching semantic Graph, manifest, and content store can +now reconstruct any named population product exactly: structural frame +artifacts restore row states and ordinary or value-revision column artifacts +restore later changes. The current HTML renderer attaches structural frames for +its weight and mass views; it does not embed complete reconstructed data values. +The current receipt exposes only the realized maximum weight ratio, not the +distribution's samples or bins. Missing evidence is identified in the page +wherever it cannot be reconstructed faithfully. diff --git a/docs/graph-interface.lock b/docs/graph-interface.lock index c3356b7cb..51e650dcb 100644 --- a/docs/graph-interface.lock +++ b/docs/graph-interface.lock @@ -1,2 +1,2 @@ -8229270f3328f537c8f8e83d6c81af39aa75d4ca5aa6a7bd3d37ecfc25aee2fe decl.py -07691fb5cf45ae700a258713ebdc9aa845891a432ec9107e223b8c943b5f3cb6 kernel.py +16830aab3e480802f086ec00367a08a0bd75e4c823dae3b3e9646207311026be decl.py +a078a8e94a900b9405e13f3646d891e0f42c45a2a998ef6937979865ff678036 kernel.py diff --git a/docs/graph-storage-benchmark.md b/docs/graph-storage-benchmark.md new file mode 100644 index 000000000..51549326e --- /dev/null +++ b/docs/graph-storage-benchmark.md @@ -0,0 +1,34 @@ +# Graph value-storage benchmark + +This benchmark measures the content-store payloads produced by the synthetic +US post-transfer parity fixture. The fixture uses the US entity schema and +executes the US graph kernels over a small, reviewable population. + +Run it with: + +```shell +uv run --package microcosm-graph pytest \ + packages/microcosm-graph/tests/test_acceptance_h_parity.py::test_h3_us_post_transfer_parity +``` + +The 2026-09-08 result on `codex/standardize-runtime-graph` is: + +| Stored values | Payload bytes | +|---|---:| +| Structural frame | 110,180 | +| Duplicate standalone copies of its coordinates (comparison) | 120,176 | +| Metadata-only coordinate references (implemented) | 0 | +| Ordinary value patches retained for reconstruction | 11,221 | + +The earlier representation wrote both the 110,180-byte structural frame and +120,176 bytes of standalone coordinate payloads. The new representation keeps +the frame once and stores each coordinate artifact key as a checked reference +to that frame. This removes 52.2% of the structural value payload while +preserving the coordinate keys used by manifests and investigation tools. + +The test constructs the comparison objects from the same values and codec, so +the measurement includes repeated entity-ID arrays and nullable-value storage. +It also verifies that ordinary and value-revision outputs remain standalone, +content-validated patches. Reference loading checks the coordinate identity, +structural-frame identity, node identity, dtype, length, and entity IDs before +returning values. diff --git a/docs/runtime-graph-stack.md b/docs/runtime-graph-stack.md new file mode 100644 index 000000000..a82e87384 --- /dev/null +++ b/docs/runtime-graph-stack.md @@ -0,0 +1,84 @@ +# Runtime graph stack ownership and source identity + +This document records the development boundary for the +`standardize-runtime-graph` change. It is an implementation note for the +stacked pull request, not a second graph specification. + +## Stack ownership + +The child branch was created from draft PR #873 at commit +`ee617ea672a7a81b4a86b3c03091a29f084682d4`. Its pull request base is Max +Ghenis's `model-artifact-graph-20260904` branch. The child branch is +`codex/standardize-runtime-graph` and its push upstream must be +`origin/codex/standardize-runtime-graph`. + +No push command for this change may name or force-update +`model-artifact-graph-20260904`. That branch is an immutable dependency of the +child changes. + +When PR #873 changes: + +1. fetch `origin/model-artifact-graph-20260904` and record its new commit; +2. rebase the independently owned child branch onto that commit; +3. resolve only conflicts in the child changes, leaving upstream conflicts + with `main` for PR #873's owner; +4. update the recorded commit in this document and the child pull request; +5. rerun typed-artifact characterization and the complete child validation + suite; and +6. push the rebased child branch with a lease that names only + `origin/codex/standardize-runtime-graph`. + +After PR #873 merges, rebase the child branch onto the resulting `main` +commit, confirm that the semantic diff relative to #873 is unchanged, rerun +validation, and change the child pull request base to `main`. The child branch +remains the only branch that this work pushes. + +## Typed-artifact baseline supplied by PR #873 + +The recorded PR #873 commit supplies the following interfaces: + +- `ArtifactType`, `ArtifactInput`, and `ArtifactOutput` declarations; +- compiler-validated typed byte dependencies between nodes, including nodes + attached to different population states; +- typed input identities in consuming node keys; +- immutable `ArtifactValue` objects in the kernel context; +- typed input and output descriptors in node receipts and schema-version 3 + manifests; +- content-store loading and integrity verification before a consumer runs; +- lossless Graph JSON serialization of typed declarations; +- separate QRF training and application kernels; and +- stable keyed randomness based on entity identifiers and draw coordinates. + +The focused tests in `test_artifact_edges.py`, `test_graph_models.py`, and +`test_keyed_randomness.py` characterize these behaviors. The runtime-graph +change builds on those interfaces rather than replacing them. + +## Reviewed source identities and runtime content identities + +PR #853 defines reviewed identities for the exact file or archive boundary +named by a source manifest and, where applicable, a Chronicle registration. +Those reviewed records are repository configuration. The graph runtime must +refer to them without copying their digest values into graph YAML. + +The graph runtime calculates a separate path-independent identity for the +bytes supplied to a `SourceRef`. A regular file is identified by its complete +byte sequence. A directory source is identified by the deterministic sequence +of relative file names and file bytes accepted by its declared codec. The +calculated identity is an execution fact and is recorded in the run manifest. + +The two identities have different purposes and may cover different byte +boundaries: + +| Record | Byte boundary | Authority | Runtime action | +| --- | --- | --- | --- | +| Reviewed expected identity | The file, archive, or archive member declared by the source manifest | PR #853 source metadata | Verify before the first consumer runs | +| Calculated graph identity | The complete file or deterministic directory payload bound to `SourceRef` | `microcosm.graph` source codec | Use in node identity and record in the run manifest | +| Codec implementation identity | The implementation that decodes the bound payload | Registered graph codec | Use in every consuming node identity | + +A source binding therefore records the reviewed expectation and the calculated +execution identity as distinct fields. If their boundaries are the same, the +runtime compares the digests directly. If a reviewed archive contains the +runtime member or extracted directory, the source metadata must declare that +relationship explicitly; the runtime must not treat unrelated hashes as equal. +Graph YAML references the reviewed source record by stable identifier and does +not maintain an independent copy of its digest. diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/yaml12.py b/packages/microcosm-build/src/microcosm/build/spec_engine/yaml12.py index 08ec91a5a..64defa58e 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/yaml12.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/yaml12.py @@ -1,380 +1,57 @@ -"""Parse the spec engine's deterministic YAML 1.2 subset. - -PyYAML intentionally defaults to YAML 1.1 scalar resolution. This module -gives the compiler a deliberately smaller surface: YAML 1.2 core scalars, -JSON-compatible values, no explicit tags or merge keys, and exactly one -document. It composes before constructing so duplicate keys and recursive -aliases cannot be hidden by Python ``dict`` construction. -""" +"""Compatibility wrappers for the graph package's strict YAML parser.""" from __future__ import annotations -import json -import math -import re -from collections.abc import Iterator from os import PathLike from pathlib import Path -import yaml -from yaml.composer import ComposerError -from yaml.error import Mark, YAMLError -from yaml.loader import SafeLoader -from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode -from yaml.tokens import DirectiveToken, TagToken - -from .errors import SpecParseError - -type JSONScalar = None | bool | int | float | str -type JSONValue = JSONScalar | list[JSONValue] | dict[str, JSONValue] - -_BOOL_TAG = "tag:yaml.org,2002:bool" -_FLOAT_TAG = "tag:yaml.org,2002:float" -_INT_TAG = "tag:yaml.org,2002:int" -_MERGE_TAG = "tag:yaml.org,2002:merge" -_NULL_TAG = "tag:yaml.org,2002:null" -_STR_TAG = "tag:yaml.org,2002:str" -_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" -_JSON_SCALAR_TAGS = frozenset({_BOOL_TAG, _FLOAT_TAG, _INT_TAG, _NULL_TAG, _STR_TAG}) - - -class _StrictYAML12Loader(SafeLoader): - """SafeLoader with YAML 1.2 core boolean/number resolution.""" - - -# Resolver tables are mutable class state. Copy both levels before removing -# YAML 1.1's bool/int/float patterns so importing this module cannot alter -# ``yaml.safe_load`` elsewhere in Microcosm. -_StrictYAML12Loader.yaml_implicit_resolvers = { - first: list(resolvers) - for first, resolvers in SafeLoader.yaml_implicit_resolvers.items() -} -for _first, _resolvers in tuple(_StrictYAML12Loader.yaml_implicit_resolvers.items()): - _StrictYAML12Loader.yaml_implicit_resolvers[_first] = [ - (tag, regexp) - for tag, regexp in _resolvers - if tag not in {_BOOL_TAG, _FLOAT_TAG, _INT_TAG} - ] - -_StrictYAML12Loader.add_implicit_resolver( - _BOOL_TAG, - re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), - list("tTfF"), +from microcosm.graph.source_errors import GraphSourceParseError +from microcosm.graph.yaml12 import ( + JSONScalar, + JSONValue, ) -_StrictYAML12Loader.add_implicit_resolver( - _INT_TAG, - re.compile( - r"^(?:[-+]?0b[0-1_]+|[-+]?0o[0-7_]+|" - r"[-+]?0x[0-9a-fA-F_]+|[-+]?[0-9][0-9_]*)$" - ), - list("-+0123456789"), +from microcosm.graph.yaml12 import ( + load_json_strict as _load_json_strict, ) -_StrictYAML12Loader.add_implicit_resolver( - _FLOAT_TAG, - re.compile( - r"^(?:" - r"[-+]?(?:[0-9][0-9_]*\.[0-9_]*|\.[0-9_]+)" - r"(?:[eE][-+]?[0-9]+)?|" - r"[-+]?[0-9][0-9_]*(?:[eE][-+]?[0-9]+)|" - r"[-+]?\.(?:inf|Inf|INF)|[-+]?\.(?:nan|NaN|NAN)" - r")$" - ), - list("-+0123456789."), +from microcosm.graph.yaml12 import ( + load_yaml12 as _load_yaml12, ) - -def _construct_yaml12_int(loader: SafeLoader, node: ScalarNode) -> int: - value = loader.construct_scalar(node).replace("_", "") - sign = -1 if value.startswith("-") else 1 - unsigned = value[1:] if value[:1] in {"+", "-"} else value - if unsigned.startswith("0b"): - return sign * int(unsigned[2:], 2) - if unsigned.startswith("0o"): - return sign * int(unsigned[2:], 8) - if unsigned.startswith("0x"): - return sign * int(unsigned[2:], 16) - return sign * int(unsigned, 10) - - -_StrictYAML12Loader.add_constructor(_INT_TAG, _construct_yaml12_int) +from .errors import SpecParseError -def _error( - message: str, - *, - source: str, - mark: Mark | None = None, -) -> SpecParseError: +def _compat(error: GraphSourceParseError) -> SpecParseError: return SpecParseError( - message, - source=source, - line=None if mark is None else mark.line + 1, - column=None if mark is None else mark.column + 1, + error.message, + source=error.source, + line=error.line, + column=error.column, ) -def _marked_yaml_error(exc: YAMLError, *, source: str) -> SpecParseError: - mark = getattr(exc, "problem_mark", None) or getattr(exc, "context_mark", None) - if isinstance(exc, ComposerError) and "single document" in str(exc): - message = "multiple YAML documents are not allowed" - else: - problem = getattr(exc, "problem", None) - message = f"invalid YAML: {problem}" if problem else "invalid YAML" - return _error(message, source=source, mark=mark) - - -def _tokens(text: str, *, source: str) -> Iterator[object]: - try: - yield from yaml.scan(text, Loader=_StrictYAML12Loader) - except YAMLError as exc: - raise _marked_yaml_error(exc, source=source) from exc - - -def _reject_syntax_extensions(text: str, *, source: str) -> None: - for token in _tokens(text, source=source): - if isinstance(token, TagToken): - raise _error( - "explicit YAML tags are not allowed", - source=source, - mark=token.start_mark, - ) - if isinstance(token, DirectiveToken): - if token.name == "TAG": - raise _error( - "YAML tag directives are not allowed", - source=source, - mark=token.start_mark, - ) - if token.name == "YAML" and token.value != (1, 2): - raise _error( - "only the YAML 1.2 directive is allowed", - source=source, - mark=token.start_mark, - ) - if token.name not in {"TAG", "YAML"}: - raise _error( - "YAML directives other than %YAML 1.2 are not allowed", - source=source, - mark=token.start_mark, - ) - - -def _validate_node( - node: Node, - *, - source: str, - active: set[int], - validated: set[int], -) -> None: - identity = id(node) - if identity in active: - raise _error( - "cyclic YAML aliases are not allowed", - source=source, - mark=node.start_mark, - ) - if identity in validated: - return - - active.add(identity) - try: - if isinstance(node, ScalarNode): - if node.tag == _TIMESTAMP_TAG: - raise _error( - "timestamps and dates are not allowed", - source=source, - mark=node.start_mark, - ) - if node.tag not in _JSON_SCALAR_TAGS: - raise _error( - "only JSON-compatible scalar values are allowed", - source=source, - mark=node.start_mark, - ) - if node.tag == _FLOAT_TAG: - normalized = node.value.replace("_", "").lower() - if normalized.lstrip("+-") in {".inf", ".nan"}: - finite = False - else: - try: - finite = math.isfinite(float(normalized)) - except ValueError: - raise _error( - "invalid YAML 1.2 number", - source=source, - mark=node.start_mark, - ) from None - if not finite: - raise _error( - "non-finite numbers are not allowed", - source=source, - mark=node.start_mark, - ) - if node.tag == _INT_TAG: - normalized = node.value.replace("_", "").lstrip("+-") - if normalized.startswith(("0b", "0o", "0x")): - normalized = normalized[2:] - if not normalized: - raise _error( - "invalid YAML 1.2 number", - source=source, - mark=node.start_mark, - ) - return - - if isinstance(node, SequenceNode): - for item in node.value: - _validate_node( - item, - source=source, - active=active, - validated=validated, - ) - return - - if isinstance(node, MappingNode): - keys: dict[str, ScalarNode] = {} - for key_node, value_node in node.value: - if key_node.tag == _MERGE_TAG: - raise _error( - "YAML merge keys are not allowed", - source=source, - mark=key_node.start_mark, - ) - if not isinstance(key_node, ScalarNode) or key_node.tag != _STR_TAG: - raise _error( - "mapping keys must be strings", - source=source, - mark=key_node.start_mark, - ) - key = key_node.value - if key in keys: - raise _error( - f"duplicate mapping key {key!r}", - source=source, - mark=key_node.start_mark, - ) - keys[key] = key_node - _validate_node( - value_node, - source=source, - active=active, - validated=validated, - ) - return - - raise _error( - "only JSON-compatible YAML nodes are allowed", - source=source, - mark=node.start_mark, - ) - finally: - active.remove(identity) - validated.add(identity) - - -def _ensure_json_value(value: object, *, source: str) -> JSONValue: - """Defend the parser boundary if a future PyYAML constructor changes.""" - - if value is None or isinstance(value, str | bool | int): - return value - if isinstance(value, float): - if not math.isfinite(value): - raise _error("non-finite numbers are not allowed", source=source) - return value - if isinstance(value, list): - return [_ensure_json_value(item, source=source) for item in value] - if isinstance(value, dict): - if not all(isinstance(key, str) for key in value): - raise _error("mapping keys must be strings", source=source) - return { - key: _ensure_json_value(item, source=source) for key, item in value.items() - } - raise _error("only JSON-compatible values are allowed", source=source) - - def load_yaml12(text: str, *, source: str = "") -> JSONValue: - """Load one document from the compiler's strict YAML 1.2 subset. - - All authored-input failures are normalized to :class:`SpecParseError`. - The returned graph contains only JSON-compatible Python values. - """ - - if not isinstance(text, str): - raise TypeError("YAML input must be text") - - _reject_syntax_extensions(text, source=source) - loader = _StrictYAML12Loader(text) try: - node = loader.get_single_node() - if node is None: - return None - _validate_node( - node, - source=source, - active=set(), - validated=set(), - ) - value = loader.construct_document(node) - except SpecParseError: - raise - except YAMLError as exc: - raise _marked_yaml_error(exc, source=source) from exc - finally: - loader.dispose() - - return _ensure_json_value(value, source=source) + return _load_yaml12(text, source=source) + except GraphSourceParseError as error: + raise _compat(error) from error def load_yaml12_file(path: str | PathLike[str]) -> JSONValue: - """Read and parse a UTF-8 YAML resource, reporting its path in errors.""" - resource = Path(path) return load_yaml12(resource.read_text(encoding="utf-8"), source=str(resource)) def load_json_strict(text: str, *, source: str = "") -> JSONValue: - """Load one strict-JSON document into the same value model as load_yaml12. - - JSON is a subset of the YAML 1.2 core schema, so a resource declared as - JSON parses with the C decoder instead of the pure-Python YAML scanner. - The JSON grammar already guarantees a single document with string mapping - keys and no tags or aliases; the two refusals it does not carry — - duplicate mapping keys and the non-finite number constants — are enforced - here so this path is never more permissive than :func:`load_yaml12`. - """ - - if not isinstance(text, str): - raise TypeError("JSON input must be text") - - def _refuse_duplicate_keys( - pairs: list[tuple[str, JSONValue]], - ) -> dict[str, JSONValue]: - mapping: dict[str, JSONValue] = {} - for key, value in pairs: - if key in mapping: - raise _error(f"duplicate mapping key {key!r}", source=source) - mapping[key] = value - return mapping - - def _refuse_constant(constant: str) -> JSONValue: - raise _error("non-finite numbers are not allowed", source=source) - try: - return json.loads( - text, - object_pairs_hook=_refuse_duplicate_keys, - parse_constant=_refuse_constant, - ) - except SpecParseError: - raise - except json.JSONDecodeError as exc: - raise SpecParseError( - f"invalid JSON: {exc.msg}", - source=source, - line=exc.lineno, - column=exc.colno, - ) from exc - - -__all__ = ["JSONScalar", "JSONValue", "load_yaml12", "load_yaml12_file"] + return _load_json_strict(text, source=source) + except GraphSourceParseError as error: + raise _compat(error) from error + + +__all__ = [ + "JSONScalar", + "JSONValue", + "load_json_strict", + "load_yaml12", + "load_yaml12_file", +] diff --git a/packages/microcosm-graph/README.md b/packages/microcosm-graph/README.md index aca5ad87b..89b5b879e 100644 --- a/packages/microcosm-graph/README.md +++ b/packages/microcosm-graph/README.md @@ -1,7 +1,8 @@ # microcosm-graph -One object replaces stages, families, batches, banks, and whole-run -authority receipts: a content-addressed DAG of cell-ownership nodes. +`microcosm.graph.Graph` is the single declaration compiled and executed for one +run. Versioned YAML is the human-authored source; a generated, versioned Graph +JSON document is an optional exact serialization for evidence and interchange. A `Node` declares the slices it reads, the cells it owns, its parameters, and the kernel that computes it. Its key is the hash of that declaration, @@ -9,8 +10,10 @@ the artifact keys of its inputs, and the kernel's implementation hash. The executor projects immutable input views, runs the kernel, patches only the owned positions with a storage-preserving assignment, and memoizes every output in a content-addressed store keyed by node key. Seeds derive from -node keys. Provenance (the run manifest) is a list of node keys plus signed -human decisions and never feeds back into a key. +node keys. The graph-bound run manifest records the semantic graph identity, +authored YAML receipts, parameters, verified source bindings, node receipts, +named products, and signed human decisions. Actual dataframe values and typed +artifacts remain in `ContentStore` and are referenced by content keys. `docs/graph-acceptance.md` is the definition of done: every property there is an executable test, committed red, and the shard is finished when none @@ -28,10 +31,93 @@ Module map: | `population.py` | Immutable population versions: `Frame` + owner map + weight lineage + mass ledger | | `executor.py` | `run_graph`: projection, patching, ownership enforcement, receipts | | `manifest.py` | `RunManifest`, `NodeReceipt`, human decision records | +| `graph_source.py` | Restricted YAML loading, declared parameter binding, and deterministic module composition | +| `serialize.py` | Optional canonical, versioned Graph JSON serialization | +| `reconstruct.py` | Named population reconstruction from a Graph, manifest, and content store | +| `materialize.py` | Versioned post-run codecs and the local candidate index | +| `runner.py` | One-root helper that compiles once, executes once, saves evidence, then materializes locally | | `view.py` | `describe(node)`: the one-screen view | -The shard depends on `microcosm-frame` only. Kernels that wrap fit, -calibrate, or a rules engine live in those shards and register here. +The package depends on `microcosm-frame` plus its YAML and JSON Schema parsing +libraries. It does not depend on `microcosm-build` or a country package. +Kernels that wrap fitting, calibration, or a rules engine live in their owning +packages and register here. + +## Author and run a graph + +The root YAML may contain declarations directly or list exact relative module +paths. Modules organize source text only: they combine into one `Graph` and are +never compiled or executed independently. + +```yaml +schema_version: 1 +country: example +modules: [sources.yaml, population.yaml] +parameters: + period: + type: integer + required: true +products: + - name: example.final + kind: population + target: {node: finalize} + - name: example.h5 + kind: export + target: {product: example.final} + codec: policyengine-h5 + codec_version: 1 +``` + +`load_graph_source(path, parameters=...)` parses the restricted YAML 1.2 +subset, validates the closed packaged schema, resolves the explicitly listed +modules, and freezes declared parameter values. It does not import code named +by YAML, read runtime sources, expand environment variables, or access the +network. `compile_graph` derives execution order from declared dependencies, +not YAML order. + +Use `run_graph_source` when a caller wants the complete sequence in one API: + +```python +result = run_graph_source( + "graph.yaml", + parameters={"period": 2026}, + sources={"survey": survey_path}, + store=ContentStore("run/store"), + kernels=registry, + manifest_path="run/manifest.json", + graph_json_path="run/graph.json", # optional generated evidence +) +``` + +The helper selects one root, compiles one `CompiledGraph`, calls `run_graph` +once, and saves the completed manifest. Country-specific Python code registers +kernel implementations and supplies source paths; it does not define a second +execution plan. + +## Reconstruct and materialize stored products + +Population products can be reconstructed after the original process exits: + +```python +manifest = RunManifest.from_json(Path("run/manifest.json").read_text()) +frame = reconstruct_population(graph, manifest, store, "example.final") +``` + +This needs no source path, kernel registry, or kernel execution. It restores +structural frames and applies stored ordinary and value-revision column patches +in canonical execution order through the product's declared node. Structural +coordinate keys refer to their one stored frame instead of duplicating its +values; ordinary patches remain standalone exact-value objects. See +`docs/graph-storage-benchmark.md` for the US fixture measurement. + +Large compatibility products are written only after a complete manifest has +been saved. A `MaterializerRegistry` binds an exact declared codec/version to a +deterministic local writer and its implementation hash. `materialize_products` +loads named stored values, writes below the supplied candidate directory, and +writes `candidate-index.json` last with the manifest identity, graph identity, +codec identity, and output content identities. It has no publication behavior; +uploading files, modifying remote release references, or sending notifications +belongs to a separate command. ## Reusable typed artifacts @@ -48,10 +134,10 @@ producer declares their type. The compiler adds these edges to dependency ordering, cycle checks, and gate ancestry. A change to recipient inputs can reuse the fitted producer. Typed -node cache records use schema 2; runs carrying typed edges or outputs use -manifest schema 3. Legacy nodes omit the new empty declarations from keys and -JSON, and legacy runs retain manifest schema 2. Cache reload validates the same -contracts as fresh execution. +node cache records use schema 2. Graph-bound runs use manifest schema 4; older +typed runs remain readable as schema 3. Legacy nodes omit the new empty +declarations from keys and JSON, and legacy runs retain manifest schema 2. +Cache reload validates the same contracts as fresh execution. Numeric contracts are deliberately restrictive: bitwise artifacts permit any consumer class; platform-bitwise artifacts require platform-bitwise consumers; diff --git a/packages/microcosm-graph/pyproject.toml b/packages/microcosm-graph/pyproject.toml index 702237838..b91650439 100644 --- a/packages/microcosm-graph/pyproject.toml +++ b/packages/microcosm-graph/pyproject.toml @@ -7,6 +7,9 @@ requires-python = ">=3.13" dependencies = [ "numpy>=2", "pandas>=2.3", + "pyyaml>=6", + "jsonschema>=4.23,<5", + "referencing>=0.35,<1", "microcosm-frame>=0.1,<0.2", ] @@ -17,6 +20,7 @@ microcosm-frame = { workspace = true } dev = [ "pytest>=8", "hypothesis>=6", + "h5py>=3", ] [build-system] diff --git a/packages/microcosm-graph/src/microcosm/graph/__init__.py b/packages/microcosm-graph/src/microcosm/graph/__init__.py index c66bac7fb..08d88732c 100644 --- a/packages/microcosm-graph/src/microcosm/graph/__init__.py +++ b/packages/microcosm-graph/src/microcosm/graph/__init__.py @@ -8,6 +8,8 @@ from importlib import metadata as _metadata +__version__ = "0.1.0" + from .decl import ( DESCRIPTIVE_FIELDS, DTYPES, @@ -20,12 +22,15 @@ ArtifactOutput, ArtifactType, CompiledGraph, + ExpectedContent, Graph, GraphError, Node, Owned, Ownership, Param, + Product, + ProductKind, Slice, SourceRef, StructuralDelta, @@ -39,6 +44,15 @@ StoreMissError, StoreUnavailableError, ) +from .graph_source import ( + GRAPH_SOURCE_SCHEMA_VERSION, + GraphSourceReceipt, + LoadedGraphSource, + compiled_graph_from_yaml_file, + graph_from_yaml_file, + load_graph_source, + validate_kernel_registry, +) from .kernel import ( ArtifactValue, Capabilities, @@ -55,14 +69,25 @@ Tolerance, source_hash, ) -from .keys import platform_fingerprint +from .keys import graph_key, platform_fingerprint from .randomness import keyed_uniform +from .source_errors import ( + GraphParameterBindingError, + GraphSourceCompositionError, + GraphSourceError, + GraphSourceParseError, + GraphSourceSchemaError, + GraphSourceValidationError, +) +from .yaml12 import load_json_strict, load_yaml12, load_yaml12_file, parse_yaml12 __all__ = [ "ArtifactInput", "ArtifactOutput", "ArtifactType", "ArtifactValue", + "BoundSource", + "GRAPH_SOURCE_SCHEMA_VERSION", "keyed_uniform", "platform_fingerprint", "DESCRIPTIVE_FIELDS", @@ -77,16 +102,29 @@ "ContentStore", "Decision", "Determinism", + "ExpectedContent", "Graph", "GraphError", + "GraphParameterBindingError", "GraphRuntimeError", + "GraphSourceCompositionError", + "GraphSourceError", + "GraphSourceParseError", + "GraphSourceReceipt", + "GraphSourceSchemaError", + "GraphSourceValidationError", "Kernel", "KernelBase", "KernelContext", "KernelRegistry", "KernelResult", "KernelRole", + "LoadedGraphSource", + "GraphRunResult", "MassRecord", + "MaterializedProduct", + "Materializer", + "MaterializerRegistry", "Node", "NodeReceipt", "NodeRejected", @@ -97,9 +135,12 @@ "Param", "Population", "PopulationView", + "Product", + "ProductKind", "PopulationError", "ResumePolicy", "RunManifest", + "CandidateIndex", "SOURCE_CODECS", "SeedSource", "NumericScope", @@ -118,13 +159,28 @@ "StructuralDelta", "WeightTransition", "compile_graph", + "compiled_graph_from_yaml_file", "describe", "explain_html", "graph_from_json", + "graph_from_yaml_file", + "graph_key", + "graph_document_from_json", + "graph_document_to_json", "graph_to_json", + "load_graph_source", + "load_json_strict", + "load_yaml12", + "load_yaml12_file", "load_source", + "materialize_products", "run_graph", + "run_graph_source", + "parse_yaml12", + "reconstruct_population", "source_hash", + "validate_kernel_registry", + "__version__", ] _FRAME_SERIES = "0.1" @@ -146,6 +202,7 @@ def _check_frame_version() -> None: from .codecs import ( # noqa: E402 - check dependency series before runtime import SOURCE_CODECS, + BoundSource, SourceCodec, SourceCodecRegistry, load_source, @@ -153,8 +210,22 @@ def _check_frame_version() -> None: from .executor import NodeRejected, run_graph # noqa: E402 from .explain import explain_html # noqa: E402 from .manifest import Decision, NodeReceipt, PopulationView, RunManifest # noqa: E402 +from .materialize import ( # noqa: E402 + CandidateIndex, + MaterializedProduct, + Materializer, + MaterializerRegistry, + materialize_products, +) from .population import MassRecord, Population, PopulationError # noqa: E402 -from .serialize import graph_from_json, graph_to_json # noqa: E402 +from .reconstruct import reconstruct_population # noqa: E402 +from .runner import GraphRunResult, run_graph_source # noqa: E402 +from .serialize import ( # noqa: E402 + graph_document_from_json, + graph_document_to_json, + graph_from_json, + graph_to_json, +) from .store import ( # noqa: E402 ContentStore, ResumePolicy, diff --git a/packages/microcosm-graph/src/microcosm/graph/canonical.py b/packages/microcosm-graph/src/microcosm/graph/canonical.py index 50e6dd65c..2b0da5356 100644 --- a/packages/microcosm-graph/src/microcosm/graph/canonical.py +++ b/packages/microcosm-graph/src/microcosm/graph/canonical.py @@ -76,6 +76,9 @@ def _declaration_value(value: object) -> object: if isinstance(value, Enum): return _declaration_value(value.value) if is_dataclass(value) and not isinstance(value, type): + project = getattr(value, "normative", None) + if callable(project): + return _declaration_value(project()) return { item.name: _declaration_value(getattr(value, item.name)) for item in fields(value) diff --git a/packages/microcosm-graph/src/microcosm/graph/codecs.py b/packages/microcosm-graph/src/microcosm/graph/codecs.py index 4cf4bd3e4..dbba6a1fe 100644 --- a/packages/microcosm-graph/src/microcosm/graph/codecs.py +++ b/packages/microcosm-graph/src/microcosm/graph/codecs.py @@ -19,7 +19,9 @@ from __future__ import annotations import json +import os from collections.abc import Callable, Mapping +from dataclasses import dataclass, field from pathlib import Path from types import MappingProxyType from typing import Any @@ -29,10 +31,12 @@ from microcosm.frame import EntitySchema, Frame, LinkSpec, WeightKind, Weights +from .kernel import source_hash from .store import ContentStore, StoreUnavailable __all__ = [ "SOURCE_CODECS", + "BoundSource", "SourceCodec", "SourceCodecRegistry", "load_csv_tables", @@ -43,13 +47,73 @@ type SourceCodec = Callable[..., Frame] +@dataclass(frozen=True) +class BoundSource(os.PathLike[str]): + """A verified source path bound to exactly one declared decoder.""" + + name: str + path: Path + codec: str + codec_impl_hash: str + content_key: str + binding_key: str + receipt: Mapping[str, object] + loader: SourceCodec = field(repr=False, compare=False) + store: ContentStore | None = field(default=None, repr=False, compare=False) + + def __post_init__(self) -> None: + object.__setattr__(self, "path", Path(self.path)) + object.__setattr__(self, "receipt", MappingProxyType(dict(self.receipt))) + + def __fspath__(self) -> str: + return os.fspath(self.path) + + def __truediv__(self, child: str | os.PathLike[str]) -> Path: + return self.path / child + + def read_text(self, *args: object, **kwargs: object) -> str: + return self.path.read_text(*args, **kwargs) + + def read_bytes(self) -> bytes: + return self.path.read_bytes() + + def decode(self, codec: str | None = None) -> Frame: + """Decode through the declaration; reject a contradictory request.""" + + if codec is not None and codec != self.codec: + raise StoreUnavailable( + f"Source {self.name!r} declares codec {self.codec!r}, not {codec!r}." + ) + try: + frame = self.loader(self.path, store=self.store) + except StoreUnavailable: + raise + except ImportError as error: + raise StoreUnavailable( + f"Source codec {self.codec!r} needs an unavailable dependency." + ) from error + if not isinstance(frame, Frame): + raise TypeError( + f"Source codec {self.codec!r} returned {type(frame).__name__}, " + "not Frame." + ) + return frame + + class SourceCodecRegistry: """Named source-to-Frame loaders.""" def __init__(self) -> None: self._loaders: dict[str, SourceCodec] = {} + self._identities: dict[str, str] = {} - def register(self, name: str, loader: SourceCodec) -> SourceCodec: + def register( + self, + name: str, + loader: SourceCodec, + *, + implementation_hash: str | None = None, + ) -> SourceCodec: """Register and return ``loader`` under a non-empty codec name.""" if not isinstance(name, str) or not name: @@ -59,7 +123,19 @@ def register(self, name: str, loader: SourceCodec) -> SourceCodec: incumbent = self._loaders.get(name) if incumbent is not None and incumbent is not loader: raise ValueError(f"Source codec {name!r} is already registered.") + identity = ( + source_hash(loader) if implementation_hash is None else implementation_hash + ) + if ( + not isinstance(identity, str) + or len(identity) != 64 + or any(character not in "0123456789abcdef" for character in identity) + ): + raise ValueError( + "Source codec implementation hashes must be SHA-256 values." + ) self._loaders[name] = loader + self._identities[name] = identity return loader def get(self, name: str) -> SourceCodec: @@ -96,6 +172,12 @@ def load( ) return frame + def implementation_hash(self, name: str) -> str: + """Return the immutable implementation identity for ``name``.""" + + self.get(name) + return self._identities[name] + def names(self) -> tuple[str, ...]: """Registered names in canonical order.""" @@ -379,11 +461,13 @@ def load_csv_tables(path: Path, *, store: ContentStore | None = None) -> Frame: def load_source( codec: str, - path: Path, + path: Path | BoundSource, *, store: ContentStore | None = None, registry: SourceCodecRegistry = SOURCE_CODECS, ) -> Frame: """Decode one source through the selected registry.""" + if isinstance(path, BoundSource): + return path.decode(codec) return registry.load(codec, path, store=store) diff --git a/packages/microcosm-graph/src/microcosm/graph/decl.py b/packages/microcosm-graph/src/microcosm/graph/decl.py index b6134c210..8e35252bb 100644 --- a/packages/microcosm-graph/src/microcosm/graph/decl.py +++ b/packages/microcosm-graph/src/microcosm/graph/decl.py @@ -60,12 +60,15 @@ "ROWS_ALL", "WEIGHT_KINDS", "CompiledGraph", + "ExpectedContent", "Graph", "GraphError", "Node", "Owned", "Ownership", "Param", + "Product", + "ProductKind", "Slice", "SourceRef", "StructuralDelta", @@ -73,7 +76,9 @@ "compile_graph", ] -type Param = bool | int | float | str | None | tuple["Param", ...] +type Param = ( + bool | int | float | str | None | tuple["Param", ...] | Mapping[str, "Param"] +) #: Fields that never enter a node key. Everything else on a declaration is #: normative. @@ -124,22 +129,44 @@ class StructuralDelta(StrEnum): FILTER = "filter" EXPAND = "expand" REWEIGHT = "reweight" + REVISION = "revision" + UNION = "union" + + +class ProductKind(StrEnum): + """Kinds of stable outputs that a graph can expose by name.""" + POPULATION = "population" + COORDINATE = "coordinate" + WEIGHTS = "weights" + ARTIFACT = "artifact" + VALIDATION = "validation" + EXPORT = "export" -def _check_param(name: str, value: object) -> None: + +def _freeze_param(name: str, value: object) -> Param: if value is None or isinstance(value, bool | int | str): - return + return value if isinstance(value, float): if not math.isfinite(value): raise GraphError(f"Parameter {name!r} is not finite: {value!r}.") - return - if isinstance(value, tuple): - for index, item in enumerate(value): - _check_param(f"{name}[{index}]", item) - return + return value + if isinstance(value, list | tuple): + return tuple( + _freeze_param(f"{name}[{index}]", item) for index, item in enumerate(value) + ) + if isinstance(value, Mapping): + frozen: dict[str, Param] = {} + if any(not isinstance(key, str) or not key for key in value): + raise GraphError( + f"Parameter mapping {name!r} requires non-empty string keys." + ) + for key in sorted(value): + frozen[key] = _freeze_param(f"{name}.{key}", value[key]) + return MappingProxyType(frozen) raise GraphError( f"Parameter {name!r} has type {type(value).__name__}; parameters are " - "bool, int, float, str, None, or tuples of those." + "finite recursively immutable JSON values." ) @@ -166,16 +193,72 @@ class SourceRef: Attributes: name: The name nodes refer to. codec: How bytes become a table (a codec registered with the store). + content_type: Media type of the bytes at the bound source boundary. + access: Optional reviewed access classification. + expected: Reviewed content identities that the executor verifies. description: Descriptive; never hashed. """ name: str codec: str description: str = "" + content_type: str = "application/octet-stream" + access: str | None = None + expected: tuple[ExpectedContent, ...] = () def __post_init__(self) -> None: _nonempty("SourceRef.name", self.name) _nonempty("SourceRef.codec", self.codec) + _nonempty("SourceRef.content_type", self.content_type) + if self.access is not None: + _nonempty("SourceRef.access", self.access) + if not isinstance(self.expected, tuple) or any( + not isinstance(item, ExpectedContent) for item in self.expected + ): + raise GraphError("SourceRef.expected must be ExpectedContent values.") + identities = [(item.boundary, item.path) for item in self.expected] + if len(set(identities)) != len(identities): + raise GraphError( + f"SourceRef {self.name!r} repeats an expected content boundary." + ) + + +@dataclass(frozen=True) +class ExpectedContent: + """One reviewed SHA-256 identity at an explicit source byte boundary.""" + + sha256: str + boundary: str = "source" + path: str | None = None + size: int | None = None + identity_ref: str = "" + + def __post_init__(self) -> None: + if ( + not isinstance(self.sha256, str) + or len(self.sha256) != 64 + or any(character not in "0123456789abcdef" for character in self.sha256) + ): + raise GraphError("ExpectedContent.sha256 must be 64 lowercase hex digits.") + if self.boundary not in {"source", "member"}: + raise GraphError("ExpectedContent.boundary must be 'source' or 'member'.") + if self.boundary == "member": + if ( + not isinstance(self.path, str) + or not self.path + or "\\" in self.path + or self.path.startswith("/") + or any(part in {"", ".", ".."} for part in self.path.split("/")) + ): + raise GraphError( + "Member ExpectedContent.path must be a safe relative POSIX path." + ) + elif self.path is not None: + raise GraphError("Source-bound ExpectedContent may not declare path.") + if self.size is not None and (type(self.size) is not int or self.size < 0): + raise GraphError("ExpectedContent.size must be a non-negative integer.") + if not isinstance(self.identity_ref, str): + raise GraphError("ExpectedContent.identity_ref must be a string.") @dataclass(frozen=True) @@ -302,6 +385,7 @@ class WeightTransition: entity: str to_kind: str mass: str = "conserve" + anchor: str | None = None def __post_init__(self) -> None: _name("WeightTransition.entity", self.entity) @@ -315,6 +399,72 @@ def __post_init__(self) -> None: f"WeightTransition.mass {self.mass!r} is not one of " f"{sorted(MASS_POLICIES)}." ) + if self.anchor is not None: + _nonempty("WeightTransition.anchor", self.anchor) + + def normative(self) -> dict[str, object]: + payload: dict[str, object] = { + "entity": self.entity, + "to_kind": self.to_kind, + "mass": self.mass, + } + if self.anchor is not None: + payload["anchor"] = self.anchor + return payload + + +@dataclass(frozen=True) +class Product: + """One stable name for a population, value, artifact, outcome, or export.""" + + name: str + kind: ProductKind + node: str | None = None + entity: str | None = None + column: str | None = None + artifact: str | None = None + source: str | None = None + codec: str | None = None + codec_version: int | None = None + + def __post_init__(self) -> None: + _nonempty("Product.name", self.name) + if not isinstance(self.kind, ProductKind): + raise GraphError("Product.kind must be a ProductKind.") + for field_name in ("node", "entity", "column", "artifact", "source", "codec"): + value = getattr(self, field_name) + if value is not None: + _nonempty(f"Product.{field_name}", value) + expected: dict[ProductKind, set[str]] = { + ProductKind.POPULATION: {"node"}, + ProductKind.COORDINATE: {"node", "entity", "column"}, + ProductKind.WEIGHTS: {"node", "entity"}, + ProductKind.ARTIFACT: {"node", "artifact"}, + ProductKind.VALIDATION: {"node"}, + ProductKind.EXPORT: {"source", "codec", "codec_version"}, + } + supplied = { + name + for name in ( + "node", + "entity", + "column", + "artifact", + "source", + "codec", + "codec_version", + ) + if getattr(self, name) is not None + } + if supplied != expected[self.kind]: + raise GraphError( + f"Product {self.name!r} kind {self.kind.value!r} requires exactly " + f"{sorted(expected[self.kind])}; got {sorted(supplied)}." + ) + if self.codec_version is not None and ( + type(self.codec_version) is not int or self.codec_version < 1 + ): + raise GraphError("Product.codec_version must be a positive integer.") @dataclass(frozen=True) @@ -362,6 +512,7 @@ class Node: population: str | None = None structural: StructuralDelta = StructuralDelta.NONE base: str | None = None + bases: tuple[str, ...] = () sources: tuple[str, ...] = () weights: WeightTransition | None = None mass: str = "conserve" @@ -370,6 +521,7 @@ class Node: entrants: bool = False artifact_inputs: tuple[ArtifactInput, ...] = () artifact_outputs: tuple[ArtifactOutput, ...] = () + requires_success: tuple[str, ...] = () def __post_init__(self) -> None: _nonempty("Node.id", self.id) @@ -387,20 +539,41 @@ def __post_init__(self) -> None: ) if len({item.name for item in declarations}) != len(declarations): raise GraphError(f"Node {self.id!r}: duplicate names in {name}.") + if not isinstance(self.requires_success, tuple) or any( + not isinstance(name, str) or not name for name in self.requires_success + ): + raise GraphError( + f"Node {self.id!r}: requires_success must contain product names." + ) + if len(set(self.requires_success)) != len(self.requires_success): + raise GraphError(f"Node {self.id!r}: requires_success contains duplicates.") if not isinstance(self.structural, StructuralDelta): raise GraphError(f"Node {self.id!r}: structural must be a StructuralDelta.") if self.mass not in MASS_POLICIES: raise GraphError(f"Node {self.id!r}: mass {self.mass!r} is not legal.") + frozen_params: dict[str, Param] = {} for name in sorted(self.params): _nonempty("Node.params key", name) - _check_param(name, self.params[name]) - object.__setattr__(self, "params", MappingProxyType(dict(self.params))) + frozen_params[name] = _freeze_param(name, self.params[name]) + object.__setattr__(self, "params", MappingProxyType(frozen_params)) + if not isinstance(self.bases, tuple): + raise GraphError(f"Node {self.id!r}: bases must be a tuple.") + for base in self.bases: + _nonempty("Node.bases[]", base) + if len(set(self.bases)) != len(self.bases): + raise GraphError(f"Node {self.id!r}: bases contains duplicates.") + if self.structural is StructuralDelta.UNION: + if len(self.bases) < 2: + raise GraphError(f"Node {self.id!r}: UNION needs at least two bases.") + object.__setattr__(self, "bases", tuple(sorted(self.bases))) + elif self.bases: + raise GraphError(f"Node {self.id!r}: only UNION declares bases.") if len({(o.entity, o.column) for o in self.outputs}) != len(self.outputs): raise GraphError(f"Node {self.id!r} declares the same owned cell twice.") if len(set(self.sources)) != len(self.sources): raise GraphError(f"Node {self.id!r} repeats a source.") if self.structural is StructuralDelta.CREATE: - if self.base is not None or self.population is not None: + if self.base is not None or self.bases or self.population is not None: raise GraphError( f"Node {self.id!r}: a CREATE node has no base or population." ) @@ -413,6 +586,28 @@ def __post_init__(self) -> None: f"Node {self.id!r}: a CREATE node must declare every column " "it loads, so ownership is total from the first node." ) + elif self.structural is StructuralDelta.UNION: + if self.base is not None or self.population is not None: + raise GraphError( + f"Node {self.id!r}: UNION declares bases, not base or population." + ) + if self.inputs or self.outputs or self.sources or self.weights is not None: + raise GraphError( + f"Node {self.id!r}: UNION is executor-owned and declares only bases." + ) + elif self.structural is StructuralDelta.REVISION: + if self.base is None or self.population is not None: + raise GraphError( + f"Node {self.id!r}: REVISION declares exactly one base." + ) + if not self.outputs or any(not output.rewrite for output in self.outputs): + raise GraphError( + f"Node {self.id!r}: REVISION outputs must be non-empty rewrites." + ) + if self.mass != "conserve" or self.weights is not None: + raise GraphError( + f"Node {self.id!r}: REVISION cannot change mass or weights." + ) elif self.structural is not StructuralDelta.NONE: if self.base is None: raise GraphError( @@ -483,7 +678,8 @@ def normative(self) -> dict[str, object]: for f in fields(self) if f.name not in DESCRIPTIVE_FIELDS and not ( - f.name in {"artifact_inputs", "artifact_outputs"} + f.name + in {"artifact_inputs", "artifact_outputs", "bases", "requires_success"} and not getattr(self, f.name) ) } @@ -510,6 +706,7 @@ class Graph: sources: tuple[SourceRef, ...] nodes: tuple[Node, ...] mass_partition: tuple[str, str] | None = None + products: tuple[Product, ...] = () def __post_init__(self) -> None: _nonempty("Graph.country", self.country) @@ -517,6 +714,12 @@ def __post_init__(self) -> None: raise GraphError("Graph repeats a source name.") if len({n.id for n in self.nodes}) != len(self.nodes): raise GraphError("Graph repeats a node id.") + if not isinstance(self.products, tuple) or any( + not isinstance(product, Product) for product in self.products + ): + raise GraphError("Graph.products must be a tuple of Product values.") + if len({product.name for product in self.products}) != len(self.products): + raise GraphError("Graph repeats a product name.") if self.mass_partition is not None: if ( not isinstance(self.mass_partition, tuple) @@ -561,6 +764,7 @@ class CompiledGraph: owners: Mapping[tuple[str, str, str], str] predecessors: Mapping[str, tuple[str, ...]] versions: Mapping[str, str] + product_nodes: Mapping[str, str] def compile_graph(graph: Graph) -> CompiledGraph: @@ -588,7 +792,15 @@ def compile_graph(graph: Graph) -> CompiledGraph: if name not in source_names: raise GraphError(f"Node {node.id!r} reads unknown source {name!r}.") if node.structural is not StructuralDelta.NONE: - if node.base is not None: + if node.structural is StructuralDelta.UNION: + for base_id in node.bases: + base = by_id.get(base_id) + if base is None or base.structural is StructuralDelta.NONE: + raise GraphError( + f"Node {node.id!r}: union base {base_id!r} is not a " + "structural node." + ) + elif node.base is not None: base = by_id.get(node.base) if base is None or base.structural is StructuralDelta.NONE: raise GraphError( @@ -669,6 +881,16 @@ def declared_dtype(version: str, entity: str, column: str) -> str | None: holder = by_id[version] if holder.structural is StructuralDelta.CREATE: return None + if holder.structural is StructuralDelta.UNION: + candidates = { + declared_dtype(base, entity, column) for base in holder.bases + } + if len(candidates) > 1: + raise GraphError( + f"UNION node {holder.id!r} has incompatible declarations " + f"for {entity}.{column}: {sorted(candidates, key=str)!r}." + ) + return next(iter(candidates)) version = holder.base # type: ignore[assignment] def reader_of(node_id: str, version: str, entity: str, column: str) -> str: @@ -679,7 +901,13 @@ def reader_of(node_id: str, version: str, entity: str, column: str) -> str: f"version {version!r} or its bases." ) owner = owners.get((version, entity, column)) - return owner if owner is not None else version + if owner is not None: + return owner + holder = by_id[version] + if holder.structural is StructuralDelta.REVISION: + assert holder.base is not None + return reader_of(node_id, holder.base, entity, column) + return version def check_mask(node_id: str, version: str, entity: str, mask: str) -> None: if mask == ROWS_ALL: @@ -701,6 +929,11 @@ def check_mask(node_id: str, version: str, entity: str, mask: str) -> None: if node.structural is StructuralDelta.CREATE: continue if node.structural is not StructuralDelta.NONE: + if node.structural is StructuralDelta.UNION: + for base in node.bases: + predecessors[node.id].add(base) + predecessors[node.id].update(members.get(base, ())) + continue base = node.base assert base is not None predecessors[node.id].add(base) @@ -711,6 +944,22 @@ def check_mask(node_id: str, version: str, entity: str, mask: str) -> None: reader_of(node.id, base, s.entity, column) ) check_mask(node.id, base, s.entity, s.rows) + if node.structural is StructuralDelta.REVISION: + for output in node.outputs: + check_mask(node.id, base, output.entity, output.rows) + base_dtype = declared_dtype(base, output.entity, output.column) + if base_dtype is None: + raise GraphError( + f"REVISION node {node.id!r} rewrites " + f"{output.entity}.{output.column}, which its base does " + "not define." + ) + if base_dtype != output.dtype: + raise GraphError( + f"REVISION node {node.id!r} declares " + f"{output.entity}.{output.column} as {output.dtype!r}; " + f"its base declares {base_dtype!r}." + ) continue version = versions[node.id] predecessors[node.id].add(version) @@ -771,6 +1020,84 @@ def check_mask(node_id: str, version: str, entity: str, mask: str) -> None: ) predecessors[node.id].add(producer.id) + product_nodes: dict[str, str] = {} + products = {product.name: product for product in graph.products} + for product in graph.products: + if product.kind is ProductKind.EXPORT: + assert product.source is not None + source_product = products.get(product.source) + if source_product is None or source_product.kind is ProductKind.EXPORT: + raise GraphError( + f"Export product {product.name!r} references missing or " + f"incompatible product {product.source!r}." + ) + product_nodes[product.name] = product_nodes.get( + product.source, source_product.node or "" + ) + continue + assert product.node is not None + target = by_id.get(product.node) + if target is None: + raise GraphError( + f"Product {product.name!r} references unknown node {product.node!r}." + ) + if product.kind is ProductKind.COORDINATE: + assert product.entity is not None and product.column is not None + version = versions[target.id] + if declared_dtype(version, product.entity, product.column) is None: + raise GraphError( + f"Product {product.name!r} references unknown coordinate " + f"{product.entity}.{product.column} at {target.id!r}." + ) + supplier = reader_of(target.id, version, product.entity, product.column) + if supplier != target.id: + predecessors[target.id].add(supplier) + elif product.kind is ProductKind.WEIGHTS: + assert product.entity is not None + elif product.kind is ProductKind.ARTIFACT: + assert product.artifact is not None + outputs = {output.name for output in target.artifact_outputs} + if product.artifact not in outputs: + raise GraphError( + f"Product {product.name!r} references undeclared artifact " + f"{product.artifact!r} on {target.id!r}." + ) + product_nodes[product.name] = target.id + + for node in graph.nodes: + for product_name in node.requires_success: + product = products.get(product_name) + if product is None or product.kind is not ProductKind.VALIDATION: + raise GraphError( + f"Node {node.id!r} requires success from {product_name!r}, " + "which is not a validation product." + ) + producer = product_nodes[product_name] + if producer == node.id: + raise GraphError( + f"Node {node.id!r} cannot require its own validation outcome." + ) + predecessors[node.id].add(producer) + + for node in graph.nodes: + if node.weights is None or node.weights.anchor is None: + continue + anchor = products.get(node.weights.anchor) + if anchor is None or anchor.kind is not ProductKind.WEIGHTS: + raise GraphError( + f"Node {node.id!r} weight anchor {node.weights.anchor!r} is not " + "a declared weights product." + ) + if anchor.entity != node.weights.entity: + raise GraphError( + f"Node {node.id!r} weight anchor entity {anchor.entity!r} does not " + f"match transition entity {node.weights.entity!r}." + ) + anchor_node = product_nodes[anchor.name] + if anchor_node == node.id: + raise GraphError(f"Node {node.id!r} cannot anchor weights to itself.") + predecessors[node.id].add(anchor_node) + depth: dict[str, int] = {} def depth_of(node_id: str, trail: tuple[str, ...]) -> int: @@ -797,4 +1124,5 @@ def depth_of(node_id: str, trail: tuple[str, ...]) -> int: {i: tuple(sorted(p)) for i, p in predecessors.items()} ), versions=MappingProxyType(versions), + product_nodes=MappingProxyType(product_nodes), ) diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 4f825ad88..bacd181c0 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -19,7 +19,7 @@ from . import keys as graph_keys from .artifact_edges import scope_payload, typed_contracts, value_from_descriptor from .canonical import canonical_json, sha256_domain -from .codecs import SOURCE_CODECS, SourceCodecRegistry +from .codecs import SOURCE_CODECS, BoundSource, SourceCodecRegistry from .decl import ( GATE_OUTCOMES, ROWS_ALL, @@ -27,9 +27,11 @@ Node, Owned, Ownership, + ProductKind, StructuralDelta, ) from .errors import NodeRejectedError +from .graph_source import validate_kernel_registry from .kernel import ( ArtifactValue, Capabilities, @@ -40,14 +42,20 @@ Numeric, NumericScope, Tolerance, + source_hash, ) from .keys import ( _capabilities_projection, artifact_key, frame_key, + graph_key, node_key, seed, + source_binding_key, + source_content_identity, source_content_key, + union_lineage_key, + validation_outcome_key, weights_key, ) from .manifest import Decision, NodeReceipt, RunManifest @@ -60,8 +68,10 @@ mass_record_receipt, patch, restore_cached_expand, + union_populations, weight_cap_receipt, ) +from .serialize import graph_document_from_json from .store import ( ContentStore, ResumePolicy, @@ -387,6 +397,11 @@ def _context_digest(context: KernelContext) -> bytes: digest.update(entity.encode("utf-8") + b"\0") digest.update(weights.kind.value.encode("ascii") + b"\0") _update_array(digest, weights.values) + for name in sorted(context.weight_anchors): + weights = context.weight_anchors[name] + digest.update(name.encode("utf-8") + b"\0") + digest.update(weights.kind.value.encode("ascii") + b"\0") + _update_array(digest, weights.values) _update_series(digest, context.strata) for name, value in sorted(context.artifacts.items()): digest.update( @@ -475,10 +490,11 @@ def _project_context( population: Population | None, *, key: str, - sources: Mapping[str, Path], + sources: Mapping[str, BoundSource], tolerances: Mapping[tuple[str, str], Tolerance | None], numerics: Mapping[tuple[str, str], NumericScope], artifacts: Mapping[str, ArtifactValue] | None = None, + weight_anchors: Mapping[str, Weights] | None = None, ) -> KernelContext: if population is None: return KernelContext( @@ -492,6 +508,7 @@ def _project_context( tolerances=tolerances, numerics=numerics, artifacts={} if artifacts is None else artifacts, + weight_anchors={} if weight_anchors is None else weight_anchors, ) frame = population.frame @@ -584,6 +601,7 @@ def _project_context( tolerances=tolerances, numerics=numerics, artifacts={} if artifacts is None else artifacts, + weight_anchors={} if weight_anchors is None else weight_anchors, ) @@ -674,7 +692,7 @@ def _input_writers( """Return causal writer lists for explicit, rewrite, and claim reads.""" node = compiled.graph.node(node_id) - if node.structural is StructuralDelta.CREATE: + if node.structural in {StructuralDelta.CREATE, StructuralDelta.UNION}: return MappingProxyType({}) input_version = ( compiled.versions[node_id] @@ -866,6 +884,20 @@ def add(writer_id: str) -> None: if _expand_wrote_rows(holder, coordinate, receipts): add(holder.id) + if holder.structural is StructuralDelta.UNION: + for base in holder.bases: + for writer in reversed( + _writers_of( + compiled, + base, + entity, + column, + exclude_node=exclude_node, + receipts=receipts, + ) + ): + add(writer) + break if holder.structural is StructuralDelta.CREATE or holder.base is None: break version = holder.base @@ -1061,7 +1093,7 @@ def _validate_result( raise NodeRejected( f"Node {node.id!r} result.columns keys must be (entity, column) strings." ) - if node.structural is StructuralDelta.NONE: + if node.structural in {StructuralDelta.NONE, StructuralDelta.REVISION}: if got != set(expected): raise NodeRejected( f"Node {node.id!r} returned output keys {sorted(got)!r}, not exactly " @@ -1080,10 +1112,20 @@ def _validate_result( elif result.keep is not None: raise NodeRejected(f"Non-FILTER node {node.id!r} returned a keep mask.") - if node.structural not in {StructuralDelta.CREATE, StructuralDelta.EXPAND} and ( - result.frame is not None - ): + frame_operations = {StructuralDelta.CREATE, StructuralDelta.EXPAND} + if cache_hit: + frame_operations.add(StructuralDelta.UNION) + if node.structural not in frame_operations and result.frame is not None: raise NodeRejected(f"Node {node.id!r} returned a Frame outside CREATE/EXPAND.") + if node.structural is StructuralDelta.UNION: + if cache_hit and result.frame is None: + raise NodeRejected( + f"Cached UNION node {node.id!r} has no executor frame artifact." + ) + if not cache_hit and result.frame is not None: + raise NodeRejected( + f"UNION node {node.id!r} returned a Frame; the executor owns union." + ) if node.structural is StructuralDelta.CREATE and result.frame is None: raise NodeRejected(f"CREATE node {node.id!r} did not return a Frame.") if node.structural is StructuralDelta.EXPAND: @@ -1306,10 +1348,11 @@ def _apply_result( cache_hit: bool = False, mass_partition: tuple[str, str] | None = None, rewrite_coordinates: frozenset[tuple[str, str]] = frozenset(), + weight_anchor: Weights | None = None, ) -> Population: if ( mass_partition is not None - and node.structural is StructuralDelta.NONE + and node.structural in {StructuralDelta.NONE, StructuralDelta.REVISION} and any( (owned.entity, owned.column) == mass_partition for owned in node.outputs ) @@ -1323,6 +1366,25 @@ def _apply_result( assert result.frame is not None return _create_population(node, result.frame) assert population is not None + if node.structural is StructuralDelta.UNION: + if cache_hit and result.frame is not None: + for entity in population.frame.entities: + if not result.frame.table(entity).equals( + population.frame.table(entity) + ): + raise NodeRejected( + f"Cached UNION node {node.id!r} frame disagrees with its bases." + ) + for entity in population.frame.weighted_entities: + expected = population.frame.weights_for(entity) + actual = result.frame.weights_for(entity) + if actual.kind is not expected.kind or not np.array_equal( + actual.values, expected.values + ): + raise NodeRejected( + f"Cached UNION node {node.id!r} weights disagree with its bases." + ) + return population if ( cache_hit and node.structural is StructuralDelta.EXPAND @@ -1374,6 +1436,7 @@ def _apply_result( result, mass_partition=mass_partition, rewrite_coordinates=rewrite_coordinates, + weight_anchor=weight_anchor, ) except NodeRejected: raise @@ -1407,7 +1470,7 @@ def _write_node( typed_artifacts: Mapping[str, object] | None = None, ) -> tuple[dict[tuple[str, str], str], dict[str, object]]: columns: dict[tuple[str, str], tuple[pd.Series, str]] = {} - if node.structural is StructuralDelta.NONE: + if node.structural in {StructuralDelta.NONE, StructuralDelta.REVISION}: declared = {(owned.entity, owned.column): owned for owned in node.outputs} for coordinate, series in result.columns.items(): columns[coordinate] = (series, declared[coordinate].dtype) @@ -1417,23 +1480,8 @@ def _write_node( series = _series_for_column(population.frame, entity, column) columns[(entity, column)] = (series, _dtype_token(series)) - column_entries: list[dict[str, str]] = [] - manifest_artifacts: dict[tuple[str, str], str] = {} - for (entity, column), (series, token) in sorted(columns.items()): - output_key = artifact_key(key, entity, column) - store.put_column( - output_key, - series, - declared_dtype=token, - entity_ids=series.index, - node_key=key, - verify_existing=verify_existing, - ) - column_entries.append({"entity": entity, "column": column, "key": output_key}) - manifest_artifacts[(entity, column)] = output_key - stored_frame_key: str | None = None - if node.structural is not StructuralDelta.NONE: + if node.structural not in {StructuralDelta.NONE, StructuralDelta.REVISION}: stored_frame_key = frame_key(key) store.put_frame( stored_frame_key, @@ -1442,6 +1490,33 @@ def _write_node( verify_existing=verify_existing, ) + column_entries: list[dict[str, str]] = [] + manifest_artifacts: dict[tuple[str, str], str] = {} + for (entity, column), (series, token) in sorted(columns.items()): + output_key = artifact_key(key, entity, column) + if stored_frame_key is None: + store.put_column( + output_key, + series, + declared_dtype=token, + entity_ids=series.index, + node_key=key, + verify_existing=verify_existing, + ) + else: + store.put_frame_column_ref( + output_key, + frame_key=stored_frame_key, + entity=entity, + column=column, + series=series, + declared_dtype=token, + node_key=key, + verify_existing=verify_existing, + ) + column_entries.append({"entity": entity, "column": column, "key": output_key}) + manifest_artifacts[(entity, column)] = output_key + weight_entry: dict[str, str] | None = None if result.weights is not None: if node.weights is not None: @@ -1488,6 +1563,23 @@ def _write_node( ) opaque_entries.append({"name": name, "key": output_key}) + outcome_entry: str | None = None + if capabilities.role is KernelRole.GATE: + outcome_entry = validation_outcome_key(key) + store.put_json( + outcome_entry, + { + "schema_version": 1, + "node_id": node.id, + "node_key": key, + "outcome": receipt["outcome"], + **({"evidence": receipt["evidence"]} if "evidence" in receipt else {}), + }, + kind="validation-outcome", + node_key=key, + verify_existing=verify_existing, + ) + record: dict[str, object] = { "schema_version": 2 if typed_artifacts else 1, **({"typed_artifacts": dict(typed_artifacts)} if typed_artifacts else {}), @@ -1501,6 +1593,7 @@ def _write_node( "frame_key": stored_frame_key, "weight": weight_entry, "opaque": opaque_entries, + **({"outcome_key": outcome_entry} if outcome_entry is not None else {}), } store.put_json( _cache_record_key(key), @@ -1535,6 +1628,12 @@ def _require_record_shape( "weight", "opaque", } + if capabilities.role is KernelRole.GATE: + if set(raw) == required | ({"typed_artifacts"} if typed_artifacts else set()): + raise StoreMiss( + f"Cached validation node {node.id!r} predates stored outcomes." + ) + required.add("outcome_key") if typed_artifacts: required.add("typed_artifacts") if set(raw) != required: @@ -1593,6 +1692,12 @@ def _require_record_shape( f"Cached receipt capabilities for node {node.id!r} disagree with " "the registered kernel contract." ) + if capabilities.role is KernelRole.GATE and raw.get( + "outcome_key" + ) != validation_outcome_key(key): + raise StoreCorrupt( + f"Cached validation outcome identity for node {node.id!r} is malformed." + ) if node.structural is StructuralDelta.EXPAND: raw_receipt = raw["receipt"] if not isinstance(raw_receipt, Mapping): @@ -1716,6 +1821,13 @@ def _preflight_record(store: ContentStore, record: Mapping[str, object]) -> None store.load_column(str(weight["key"])) for entry in _record_entries(record, "opaque"): store.load_bytes(str(entry.get("key"))) + outcome_key = record.get("outcome_key") + if outcome_key is not None: + store.load_json(str(outcome_key), kind="validation-outcome") + receipt = record.get("receipt") + lineage = receipt.get("union_lineage") if isinstance(receipt, Mapping) else None + if isinstance(lineage, Mapping) and isinstance(lineage.get("key"), str): + store.load_json(lineage["key"], kind="union-lineage") def _load_cached_result( @@ -1740,7 +1852,7 @@ def _load_cached_result( manifest_artifacts[coordinate] = output_key result_columns: dict[tuple[str, str], pd.Series] = {} - if node.structural is StructuralDelta.NONE: + if node.structural in {StructuralDelta.NONE, StructuralDelta.REVISION}: for owned in node.outputs: coordinate = (owned.entity, owned.column) try: @@ -1754,7 +1866,14 @@ def _load_cached_result( frame_artifact = record["frame_key"] if frame_artifact is not None: loaded_frame = store.load_frame(str(frame_artifact)) - if node.structural is not StructuralDelta.NONE and loaded_frame is None: + if ( + node.structural + not in { + StructuralDelta.NONE, + StructuralDelta.REVISION, + } + and loaded_frame is None + ): raise StoreMiss(f"Cached structural node {node.id!r} has no frame artifact.") loaded_weights: Weights | None = None @@ -1799,6 +1918,25 @@ def _load_cached_result( raw_receipt = record["receipt"] if not isinstance(raw_receipt, dict): raise StoreCorrupt(f"Cached node {node.id!r} receipt is malformed.") + outcome_key = record.get("outcome_key") + if outcome_key is not None: + stored_outcome = store.load_json(str(outcome_key), kind="validation-outcome") + expected_outcome = { + "schema_version": 1, + "node_id": node.id, + "node_key": str(record["node_key"]), + "outcome": raw_receipt.get("outcome"), + **( + {"evidence": raw_receipt["evidence"]} + if "evidence" in raw_receipt + else {} + ), + } + if stored_outcome != expected_outcome: + raise StoreCorrupt( + f"Stored validation outcome for node {node.id!r} disagrees " + "with its receipt." + ) # Reapply FILTER/REWEIGHT to the current base so graph mass checks and # ledgers are reconstructed on a hit. Their stored final frame was loaded @@ -1843,7 +1981,7 @@ def _source_paths_and_keys( compiled: CompiledGraph, sources: Mapping[str, Path], store: ContentStore, -) -> tuple[dict[str, Path], dict[str, str]]: +) -> tuple[dict[str, BoundSource], dict[str, str], dict[str, Mapping[str, object]]]: declared = {source.name: source for source in compiled.graph.sources} used = {name for node in compiled.graph.nodes for name in node.sources} missing = sorted(used - sources.keys()) @@ -1852,27 +1990,117 @@ def _source_paths_and_keys( unknown = sorted(sources.keys() - declared.keys()) if unknown: raise ValueError(f"Source paths supplied for undeclared names {unknown!r}.") - resolved: dict[str, Path] = {} + resolved: dict[str, BoundSource] = {} identities: dict[str, str] = {} + receipts: dict[str, Mapping[str, object]] = {} for name in sorted(used): path = Path(sources[name]).resolve(strict=True) - # Codec availability is verified before any kernel can execute. The - # CREATE kernel remains the declared computation that invokes it. - codec = declared[name].codec + declaration = declared[name] + codec = declaration.codec configured = store.codecs if configured is None: - SOURCE_CODECS.get(codec) + loader = SOURCE_CODECS.get(codec) + codec_identity = SOURCE_CODECS.implementation_hash(codec) elif isinstance(configured, SourceCodecRegistry): - configured.get(codec) + loader = configured.get(codec) + codec_identity = configured.implementation_hash(codec) elif isinstance(configured, Mapping): loader = configured.get(codec) if not callable(loader): raise StoreUnavailable(f"Source codec {codec!r} is not installed.") + codec_identity = source_hash(loader) else: # defended by ContentStore.__init__ raise StoreUnavailable("ContentStore has an invalid codec registry.") - resolved[name] = path - identities[name] = source_content_key(name, path) - return resolved, identities + boundary_kind, calculated_sha256, calculated_size = source_content_identity( + path + ) + content_key = source_content_key(name, path) + expectation_receipts: list[dict[str, object]] = [] + for expectation in declaration.expected: + expected_path = path + if expectation.boundary == "member": + assert expectation.path is not None + expected_path = path.joinpath(*expectation.path.split("/")) + if expected_path.is_symlink() or not expected_path.is_file(): + raise StoreUnavailable( + f"Source {name!r} expected member {expectation.path!r} " + "is not a regular file." + ) + member_kind, observed_sha256, observed_size = source_content_identity( + expected_path + ) + assert member_kind == "file" + else: + observed_sha256 = calculated_sha256 + observed_size = calculated_size + matches = expectation.sha256 == observed_sha256 and ( + expectation.size is None or expectation.size == observed_size + ) + expectation_receipts.append( + { + "boundary": expectation.boundary, + **( + {"path": expectation.path} + if expectation.path is not None + else {} + ), + "expected_sha256": expectation.sha256, + **( + {"expected_size": expectation.size} + if expectation.size is not None + else {} + ), + "calculated_sha256": observed_sha256, + "calculated_size": observed_size, + **( + {"identity_ref": expectation.identity_ref} + if expectation.identity_ref + else {} + ), + "matched": matches, + } + ) + if not matches: + raise StoreUnavailable( + f"Source {name!r} content identity mismatch at " + f"{expectation.boundary!r} boundary" + + (f" {expectation.path!r}" if expectation.path is not None else "") + + f": expected {expectation.sha256}, calculated {observed_sha256}." + ) + binding_identity = source_binding_key( + content_key, codec, codec_identity, declaration.content_type + ) + receipt: Mapping[str, object] = MappingProxyType( + { + "name": name, + "content_type": declaration.content_type, + **({"access": declaration.access} if declaration.access else {}), + "content": { + "boundary": boundary_kind, + "sha256": calculated_sha256, + "size": calculated_size, + "key": content_key, + }, + "codec": codec, + "codec_impl_hash": codec_identity, + "binding_key": binding_identity, + "expected": expectation_receipts, + } + ) + resolved[name] = BoundSource( + name=name, + path=path, + codec=codec, + codec_impl_hash=codec_identity, + content_key=content_key, + binding_key=binding_identity, + receipt=receipt, + loader=loader, # type: ignore[arg-type] + store=store, + ) + identities[name] = binding_identity + receipts[name] = receipt + return resolved, identities, receipts def _all_node_keys( @@ -1910,8 +2138,21 @@ def _preflight_require( kernels: KernelRegistry, ) -> None: missing: list[str] = [] + outcomes: dict[str, str] = {} + unreached: set[str] = set() for node_id in compiled.order: node = compiled.graph.node(node_id) + required_producers = { + compiled.product_nodes[name] for name in node.requires_success + } + if any( + predecessor in unreached for predecessor in compiled.predecessors[node_id] + ) or any( + outcomes.get(producer) not in _CERTIFYING_GATE_OUTCOMES + for producer in required_producers + ): + unreached.add(node_id) + continue try: record = _load_record( store, @@ -1928,6 +2169,11 @@ def _preflight_require( exact=False, ) _preflight_record(store, record) + raw_receipt = record.get("receipt") + if kernels.get( + node.kernel + ).capabilities.role is KernelRole.GATE and isinstance(raw_receipt, Mapping): + outcomes[node_id] = str(raw_receipt.get("outcome")) except StoreMiss: missing.append(node_id) if missing: @@ -1952,6 +2198,202 @@ def _preflight_expand_declarations(compiled: CompiledGraph) -> None: ) from error +def _weight_anchors( + compiled: CompiledGraph, + node: Node, + population: Population | None, + populations: Mapping[str, Population], +) -> Mapping[str, Weights]: + transition = node.weights + if transition is None or transition.anchor is None: + return MappingProxyType({}) + if population is None: + raise NodeRejected(f"Node {node.id!r} has no population for its weight anchor.") + product_name = transition.anchor + producer = compiled.product_nodes[product_name] + version = compiled.versions[producer] + try: + anchor_population = populations[version] + anchor = anchor_population.frame.weights_for(transition.entity) + except (KeyError, ValueError) as error: + raise NodeRejected( + f"Node {node.id!r} cannot resolve weight anchor {product_name!r}." + ) from error + id_column = population.frame.schema.entity_id_column(transition.entity) + current_ids = pd.Index(population.frame.table(transition.entity)[id_column]) + anchor_ids = pd.Index(anchor_population.frame.table(transition.entity)[id_column]) + if not current_ids.equals(anchor_ids): + raise NodeRejected( + f"Node {node.id!r} weight anchor {product_name!r} is not exactly " + "aligned to the transition population." + ) + return MappingProxyType({product_name: anchor}) + + +def _weight_anchor_receipt( + compiled: CompiledGraph, + node: Node, + updated: Population, + anchors: Mapping[str, Weights], + keys: Mapping[str, str], +) -> Mapping[str, object]: + transition = node.weights + if transition is None or transition.anchor is None: + return MappingProxyType({}) + name = transition.anchor + anchor = anchors[name] + current = updated.frame.weights_for(transition.entity) + ratios = np.divide( + current.values, + anchor.values, + out=np.full(len(current.values), np.inf, dtype=np.float64), + where=anchor.values > 0, + ) + ratios[(anchor.values == 0) & (current.values == 0)] = 0.0 + realized = float(ratios.max()) + cap = node.params.get("max_weight_ratio") + if cap is not None: + if ( + isinstance(cap, bool) + or not isinstance(cap, int | float) + or not np.isfinite(float(cap)) + or float(cap) <= 0 + ): + raise NodeRejected( + f"Node {node.id!r} max_weight_ratio must be finite and positive." + ) + if realized > float(cap) and not np.isclose( + realized, float(cap), rtol=1e-12, atol=0.0 + ): + raise NodeRejected( + f"Node {node.id!r} realized weight ratio {realized!r} exceeds " + f"{float(cap)!r} relative to {name!r}." + ) + producer = compiled.product_nodes[name] + return MappingProxyType( + { + "weight_anchor": { + "product": name, + "producer": producer, + "producer_key": keys[producer], + "entity": transition.entity, + "kind": anchor.kind.value, + "realized_max_weight_ratio": realized, + **({"max_weight_ratio": float(cap)} if cap is not None else {}), + } + } + ) + + +def _blocked_by( + compiled: CompiledGraph, + node: Node, + receipts: Mapping[str, NodeReceipt], +) -> tuple[str, ...]: + """Return failed required validations or already-unreached predecessors.""" + + blockers = { + predecessor + for predecessor in compiled.predecessors[node.id] + if predecessor in receipts and receipts[predecessor].status == "unreached" + } + for product_name in node.requires_success: + producer = compiled.product_nodes[product_name] + receipt = receipts[producer] + if ( + receipt.status == "unreached" + or receipt.receipt.get("outcome") not in _CERTIFYING_GATE_OUTCOMES + ): + blockers.add(producer) + return tuple(sorted(blockers)) + + +def _graph_product_receipts( + compiled: CompiledGraph, receipts: Mapping[str, NodeReceipt] +) -> Mapping[str, object]: + """Resolve declared product names to portable producer and artifact records.""" + + products: dict[str, object] = {} + + def coordinate_supplier(version: str, entity: str, column: str) -> str: + owner = compiled.owners.get((version, entity, column)) + if owner is not None: + return owner + holder = compiled.graph.node(version) + if holder.structural is StructuralDelta.REVISION: + assert holder.base is not None + return coordinate_supplier(holder.base, entity, column) + return version + + for product in sorted(compiled.graph.products, key=lambda item: item.name): + if product.kind is ProductKind.EXPORT: + products[product.name] = { + "kind": product.kind.value, + "source": product.source, + "codec": product.codec, + "codec_version": product.codec_version, + } + continue + assert product.node is not None + producer_receipt = receipts[product.node] + record: dict[str, object] = { + "kind": product.kind.value, + "producer": product.node, + "producer_key": producer_receipt.key, + "status": producer_receipt.status, + } + if producer_receipt.status == "unreached": + products[product.name] = record + continue + if product.kind is ProductKind.POPULATION: + record["version"] = compiled.versions[product.node] + elif product.kind is ProductKind.COORDINATE: + assert product.entity is not None and product.column is not None + supplier = coordinate_supplier( + compiled.versions[product.node], product.entity, product.column + ) + record.update( + { + "entity": product.entity, + "column": product.column, + "supplier": supplier, + "key": receipts[supplier].artifacts[ + (product.entity, product.column) + ], + } + ) + elif product.kind is ProductKind.WEIGHTS: + assert product.entity is not None + record["entity"] = product.entity + version = compiled.versions[product.node] + state_receipt = receipts[version] + while ( + state_receipt.weight_key is None + and state_receipt.frame_key is None + and compiled.graph.node(version).structural is StructuralDelta.REVISION + ): + base = compiled.graph.node(version).base + assert base is not None + version = base + state_receipt = receipts[version] + record["state"] = version + if state_receipt.weight_key is not None: + record["key"] = state_receipt.weight_key + record["storage"] = "column" + else: + record["key"] = state_receipt.frame_key + record["storage"] = "frame" + elif product.kind is ProductKind.ARTIFACT: + assert product.artifact is not None + record["artifact"] = product.artifact + record["key"] = producer_receipt.opaque_artifacts[product.artifact] + elif product.kind is ProductKind.VALIDATION: + record["key"] = producer_receipt.outcome_key + record["outcome"] = producer_receipt.receipt["outcome"] + products[product.name] = record + return MappingProxyType(products) + + def run_graph( compiled: CompiledGraph, *, @@ -1960,6 +2402,8 @@ def run_graph( kernels: KernelRegistry, resume: ResumePolicy = "auto", decisions: tuple[Decision, ...] = (), + graph_source: object | None = None, + graph_json: str | bytes | None = None, ) -> RunManifest: """Execute a compiled graph with content-addressed reuse and receipts.""" @@ -1975,9 +2419,34 @@ def run_graph( raise TypeError("decisions must contain Decision records or mappings.") decisions = tuple(normalized_decisions) + source_graph = getattr(graph_source, "graph", None) + if graph_source is not None and source_graph != compiled.graph: + raise ValueError("graph_source does not describe the compiled Graph.") + graph_source_receipts = tuple( + { + "path": receipt.path, + "sha256": receipt.sha256, + } + for receipt in getattr(graph_source, "receipts", ()) + ) + run_parameters = dict(getattr(graph_source, "parameters", {})) + graph_json_key: str | None = None + if graph_json is not None: + graph_json_bytes = ( + graph_json.encode("utf-8") if isinstance(graph_json, str) else graph_json + ) + decoded = graph_json_bytes.decode("utf-8") + if graph_document_from_json(decoded) != compiled.graph: + raise ValueError("graph_json does not serialize the compiled Graph.") + graph_json_key = sha256_domain("graph-json", graph_json_bytes) + store.put_bytes(graph_json_key, graph_json_bytes) + + validate_kernel_registry(compiled, kernels) _preflight_expand_declarations(compiled) started_at = _now() - source_paths, source_keys = _source_paths_and_keys(compiled, sources, store) + source_paths, source_keys, source_receipts = _source_paths_and_keys( + compiled, sources, store + ) keys, implementations = _all_node_keys(compiled, kernels, source_keys) contracts = { node_id: typed_contracts(compiled, compiled.graph.node(node_id), keys, kernels) @@ -1999,9 +2468,46 @@ def run_graph( f"Node {node.id!r} structural declaration does not match kernel " "capabilities." ) + blockers = _blocked_by(compiled, node, receipts) + if blockers: + receipts[node_id] = NodeReceipt( + typed_artifacts={}, + key=key, + hit=False, + seed=seed(key), + kernel_ref=node.kernel, + kernel_impl_hash=implementation, + capabilities=kernel.capabilities, + receipt=MappingProxyType( + { + "blocked_by": list(blockers), + **( + {"outcome": "unreached"} + if kernel.capabilities.role is KernelRole.GATE + else {} + ), + } + ), + wall_time=time.perf_counter() - node_started, + status="unreached", + blocked_by=blockers, + ) + continue + union_lineage: Mapping[str, tuple[tuple[object, str, object], ...]] | None = ( + None + ) if node.structural is StructuralDelta.CREATE: incumbent: Population | None = None + elif node.structural is StructuralDelta.UNION: + try: + incumbent, union_lineage = union_populations( + {base: populations[base] for base in node.bases}, node + ) + except (TypeError, ValueError) as error: + raise NodeRejected( + f"UNION node {node.id!r} rejected its bases: {error}" + ) from error elif node.structural is StructuralDelta.NONE: incumbent = populations[compiled.versions[node_id]] else: @@ -2021,6 +2527,7 @@ def run_graph( numerics=input_numerics, ) tolerance_writers = _tolerance_writer_payload(input_writers) + weight_anchors = _weight_anchors(compiled, node, incumbent, populations) typed = contracts[node_id] artifact_values = {} @@ -2076,6 +2583,7 @@ def run_graph( tolerances=input_tolerances, numerics=input_numerics, artifacts=artifact_values, + weight_anchors=weight_anchors, ) before = _context_digest(context) try: @@ -2093,8 +2601,8 @@ def run_graph( if before != after: raise NodeRejected(f"Node {node.id!r} mutated its input context.") for name in node.sources: - current = source_content_key(name, source_paths[name]) - if current != source_keys[name]: + current = source_content_key(name, source_paths[name].path) + if current != source_paths[name].content_key: raise NodeRejected( f"Node {node.id!r} changed source {name!r} while running." ) @@ -2109,6 +2617,24 @@ def run_graph( _validate_entrant_materialization_contract( compiled, node, incumbent, normalized_receipt ) + if union_lineage is not None: + lineage_payload = { + entity: [list(entry) for entry in entries] + for entity, entries in union_lineage.items() + } + lineage_key = union_lineage_key(key) + store.put_json( + lineage_key, + lineage_payload, + kind="union-lineage", + node_key=key, + ) + normalized_receipt["union_lineage"] = { + "key": lineage_key, + "rows": { + entity: len(entries) for entity, entries in union_lineage.items() + }, + } if kernel.capabilities.role is KernelRole.RELEASE: derived_tier, gate_ids = _release_tier(compiled, node_id, receipts) _validate_release_tier(node, result, derived_tier) @@ -2144,6 +2670,7 @@ def run_graph( cache_hit=hit, mass_partition=compiled.graph.mass_partition, rewrite_coordinates=expand_rewrites, + weight_anchor=next(iter(weight_anchors.values()), None), ) if node.structural is StructuralDelta.EXPAND: assert incumbent is not None @@ -2183,6 +2710,7 @@ def run_graph( if node.structural not in { StructuralDelta.NONE, StructuralDelta.CREATE, + StructuralDelta.REVISION, }: existing_mass = normalized_receipt.get("mass", {}) if not isinstance(existing_mass, Mapping): # defended by mass validation @@ -2197,6 +2725,9 @@ def run_graph( ) from error normalized_receipt["mass"] = {**existing_mass, **authored_mass} normalized_receipt.update(weight_cap_receipt(updated, node)) + normalized_receipt.update( + _weight_anchor_receipt(compiled, node, updated, weight_anchors, keys) + ) cache_receipt = normalized_receipt run_receipt = dict(cache_receipt) if kernel.capabilities.role is KernelRole.RELEASE: @@ -2233,6 +2764,10 @@ def run_graph( receipt_weight_key = raw_weight["key"] else: # generated records cannot reach this branch raise StoreCorrupt(f"Node {node.id!r} weight identity is malformed.") + raw_outcome_key = record.get("outcome_key") + receipt_outcome_key = ( + str(raw_outcome_key) if raw_outcome_key is not None else None + ) receipt_opaque: dict[str, str] = {} for entry in _record_entries(record, "opaque"): name = entry.get("name") @@ -2257,6 +2792,7 @@ def run_graph( frame_key=receipt_frame_key, weight_key=receipt_weight_key, opaque_artifacts=MappingProxyType(receipt_opaque), + outcome_key=receipt_outcome_key, ) return RunManifest( @@ -2275,4 +2811,10 @@ def run_graph( for version, population in populations.items() } ), + graph_key=graph_key(compiled.graph), + graph_source_receipts=graph_source_receipts, + parameters=run_parameters, + source_bindings=MappingProxyType(source_receipts), + graph_json_key=graph_json_key, + products=_graph_product_receipts(compiled, receipts), ) diff --git a/packages/microcosm-graph/src/microcosm/graph/graph_schema.py b/packages/microcosm-graph/src/microcosm/graph/graph_schema.py new file mode 100644 index 000000000..6dcb777b1 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/graph_schema.py @@ -0,0 +1,107 @@ +"""Closed JSON Schema catalog for authored graph YAML.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from functools import lru_cache +from importlib import resources +from types import MappingProxyType +from typing import Any + +from jsonschema import Draft202012Validator +from referencing import Registry, Resource +from referencing.exceptions import NoSuchResource + +from .source_errors import GraphSourceSchemaError, GraphSourceValidationError +from .yaml12 import ParsedYAML, SourceLocation + +_SCHEMAS = ("graph-source-v1.schema.json", "graph-module-v1.schema.json") +_DRAFT = "https://json-schema.org/draft/2020-12/schema" + + +def _refuse_retrieval(uri: str) -> Resource[Any]: + raise NoSuchResource(ref=uri) + + +def _pointer(parts: object) -> str: + encoded = [str(part).replace("~", "~0").replace("/", "~1") for part in parts] + return "/" + "/".join(encoded) if encoded else "/" + + +def _location(parsed: ParsedYAML, pointer: str) -> SourceLocation | None: + candidate = pointer + while candidate not in parsed.locations and candidate not in {"", "/"}: + candidate = candidate.rsplit("/", 1)[0] or "/" + return parsed.locations.get(candidate) or parsed.locations.get("/") + + +@lru_cache(maxsize=1) +def schema_catalog() -> tuple[Mapping[str, Mapping[str, Any]], Registry[Any]]: + root = resources.files("microcosm.graph").joinpath("schema") + found = tuple( + sorted(item.name for item in root.iterdir() if item.name.endswith(".json")) + ) + if found != tuple(sorted(_SCHEMAS)): + raise GraphSourceSchemaError( + f"graph schema catalog must contain exactly {sorted(_SCHEMAS)!r}; " + f"found {list(found)!r}" + ) + schemas: dict[str, Mapping[str, Any]] = {} + registry: Registry[Any] = Registry(retrieve=_refuse_retrieval) + for filename in _SCHEMAS: + try: + value = json.loads(root.joinpath(filename).read_text(encoding="utf-8")) + Draft202012Validator.check_schema(value) + except Exception as error: + raise GraphSourceSchemaError( + f"{filename}: invalid packaged Draft 2020-12 schema: {error}" + ) from error + if value.get("$schema") != _DRAFT or value.get("$id") != filename: + raise GraphSourceSchemaError( + f"{filename}: $schema and $id must identify the packaged schema" + ) + schemas[filename] = value + registry = registry.with_resource(filename, Resource.from_contents(value)) + return MappingProxyType(schemas), registry + + +def validate_graph_document(parsed: ParsedYAML, *, module: bool) -> None: + """Validate one parsed document and report its first stable error.""" + schemas, registry = schema_catalog() + schema_id = ( + "graph-module-v1.schema.json" if module else "graph-source-v1.schema.json" + ) + validator = Draft202012Validator(schemas[schema_id], registry=registry) + errors = sorted( + validator.iter_errors(parsed.value), + key=lambda error: ( + _pointer(error.absolute_path), + _pointer(error.absolute_schema_path), + error.message, + ), + ) + if not errors: + return + error = errors[0] + pointer = _pointer(error.absolute_path) + if error.validator == "additionalProperties": + match = re.search(r"\('([^']+)' was unexpected\)", error.message) + if match: + pointer = ( + pointer.rstrip("/") + + "/" + + match.group(1).replace("~", "~0").replace("/", "~1") + ) + location = _location(parsed, pointer) + raise GraphSourceValidationError( + error.message, + source=None if location is None else location.source, + pointer=pointer, + line=None if location is None else location.line, + column=None if location is None else location.column, + ) + + +__all__ = ["schema_catalog", "validate_graph_document"] diff --git a/packages/microcosm-graph/src/microcosm/graph/graph_source.py b/packages/microcosm-graph/src/microcosm/graph/graph_source.py new file mode 100644 index 000000000..59f3d73b5 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/graph_source.py @@ -0,0 +1,525 @@ +"""Load versioned graph YAML and compose it into one declaration.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from types import MappingProxyType + +from .artifact_edges import numeric_scope, require_compatible_scope +from .decl import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + CompiledGraph, + ExpectedContent, + Graph, + GraphError, + Node, + Owned, + Ownership, + Param, + Product, + ProductKind, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, + _freeze_param, + compile_graph, +) +from .graph_schema import validate_graph_document +from .kernel import KernelRegistry +from .source_errors import ( + GraphParameterBindingError, + GraphSourceCompositionError, + GraphSourceValidationError, +) +from .yaml12 import ParsedYAML, parse_yaml12 + +GRAPH_SOURCE_SCHEMA_VERSION = 1 + + +@dataclass(frozen=True) +class GraphSourceReceipt: + """Identity of one authored YAML document used in composition.""" + + path: str + sha256: str + + +@dataclass(frozen=True) +class LoadedGraphSource: + """A Graph plus authored-input evidence retained for run manifests.""" + + graph: Graph + schema_version: int + parameters: Mapping[str, Param] + receipts: tuple[GraphSourceReceipt, ...] + products: tuple[Product, ...] = () + + +@dataclass(frozen=True) +class _Document: + path: Path + relative: str + raw: bytes + parsed: ParsedYAML + + +def _mapping(value: object, label: str) -> Mapping[str, object]: + if not isinstance(value, Mapping): + raise GraphSourceValidationError(f"{label} must be a mapping") + return value + + +def _safe_module_path(root: Path, raw: object, *, source: Path) -> Path: + if not isinstance(raw, str) or not raw or "\\" in raw: + raise GraphSourceCompositionError( + f"module path must be a non-empty relative POSIX path: {raw!r}", + source=str(source), + ) + relative = PurePosixPath(raw) + if relative.is_absolute() or any( + part in {"", ".", ".."} for part in relative.parts + ): + raise GraphSourceCompositionError( + f"unsafe module path {raw!r}", source=str(source) + ) + candidate = root.joinpath(*relative.parts) + current = root + for part in relative.parts: + current = current / part + if current.is_symlink(): + raise GraphSourceCompositionError( + f"module path may not contain a symbolic link: {raw!r}", + source=str(source), + ) + try: + candidate.relative_to(root) + except ValueError as error: + raise GraphSourceCompositionError( + f"module path escapes graph root: {raw!r}", source=str(source) + ) from error + if not candidate.is_file(): + raise GraphSourceCompositionError( + f"declared module does not exist: {raw!r}", source=str(source) + ) + return candidate + + +def _read_document(path: Path, root: Path, *, module: bool) -> _Document: + raw = path.read_bytes() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as error: + raise GraphSourceValidationError( + "graph YAML must be UTF-8", source=str(path) + ) from error + parsed = parse_yaml12(text, source=str(path)) + value = _mapping(parsed.value, "graph document") + version = value.get("schema_version") + if version != GRAPH_SOURCE_SCHEMA_VERSION: + raise GraphSourceValidationError( + f"unsupported graph schema version {version!r}", + source=str(path), + pointer="/schema_version", + ) + validate_graph_document(parsed, module=module) + return _Document(path, path.relative_to(root).as_posix(), raw, parsed) + + +def _parameter_value( + name: str, declaration: Mapping[str, object], supplied: object +) -> Param: + kind = declaration["type"] + valid = { + "boolean": lambda value: isinstance(value, bool), + "integer": lambda value: isinstance(value, int) and not isinstance(value, bool), + "number": lambda value: ( + isinstance(value, int | float) and not isinstance(value, bool) + ), + "string": lambda value: isinstance(value, str), + }[kind] + if not valid(supplied): + raise GraphParameterBindingError( + f"parameter {name!r} requires type {kind}, got {type(supplied).__name__}" + ) + if "allowed" in declaration and supplied not in declaration["allowed"]: + raise GraphParameterBindingError( + f"parameter {name!r} is not one of its allowed values" + ) + if "minimum" in declaration and supplied < declaration["minimum"]: # type: ignore[operator] + raise GraphParameterBindingError(f"parameter {name!r} is below its minimum") + if "maximum" in declaration and supplied > declaration["maximum"]: # type: ignore[operator] + raise GraphParameterBindingError(f"parameter {name!r} is above its maximum") + try: + return _freeze_param(name, supplied) + except GraphError as error: + raise GraphParameterBindingError(str(error)) from error + + +def _bind_parameters( + declarations: Mapping[str, Mapping[str, object]], supplied: Mapping[str, object] +) -> Mapping[str, Param]: + unknown = sorted(set(supplied) - set(declarations)) + if unknown: + raise GraphParameterBindingError( + f"undeclared run parameter override(s): {', '.join(unknown)}" + ) + result: dict[str, Param] = {} + for name in sorted(declarations): + declaration = declarations[name] + required = declaration["required"] + has_default = "default" in declaration + if required and has_default: + raise GraphParameterBindingError( + f"required parameter {name!r} may not declare a default" + ) + if not required and not has_default: + raise GraphParameterBindingError( + f"optional parameter {name!r} must declare a default" + ) + if name in supplied: + raw = supplied[name] + elif has_default: + raw = declaration["default"] + else: + raise GraphParameterBindingError( + f"required run parameter {name!r} was not supplied" + ) + result[name] = _parameter_value(name, declaration, raw) + return MappingProxyType(result) + + +def _artifact_type(value: object) -> ArtifactType: + item = _mapping(value, "artifact type") + return ArtifactType(str(item["name"]), int(item["schema_version"])) + + +def _lower_node(raw: Mapping[str, object], bindings: Mapping[str, Param]) -> Node: + params = { + str(name): _freeze_param(str(name), value) + for name, value in _mapping(raw.get("params", {}), "node params").items() + } + for local, global_name in _mapping( + raw.get("param_bindings", {}), "parameter bindings" + ).items(): + if local in params: + raise GraphParameterBindingError( + f"node {raw['id']!r} parameter {local!r} is both literal and bound" + ) + if global_name not in bindings: + raise GraphParameterBindingError( + f"node {raw['id']!r} binds undefined parameter {global_name!r}" + ) + params[str(local)] = bindings[str(global_name)] + inputs = tuple( + Slice( + str(item["entity"]), + tuple(str(value) for value in item["columns"]), + str(item.get("rows", "all")), + ) + for item in (_mapping(value, "slice") for value in raw.get("inputs", [])) + ) + outputs = tuple( + Owned( + str(item["entity"]), + str(item["column"]), + str(item["dtype"]), + str(item.get("rows", "all")), + Ownership(str(item.get("ownership", "produced"))), + bool(item.get("rewrite", False)), + ) + for item in ( + _mapping(value, "owned coordinate") for value in raw.get("outputs", []) + ) + ) + weight_raw = raw.get("weights") + weights = None + if weight_raw is not None: + item = _mapping(weight_raw, "weight transition") + weights = WeightTransition( + str(item["entity"]), + str(item["to_kind"]), + str(item.get("mass", "conserve")), + None if "anchor" not in item else str(item["anchor"]), + ) + artifact_inputs = tuple( + ArtifactInput( + str(item["name"]), + str(item["producer"]), + str(item["artifact"]), + _artifact_type(item["type"]), + ) + for item in ( + _mapping(value, "artifact input") + for value in raw.get("artifact_inputs", []) + ) + ) + artifact_outputs = tuple( + ArtifactOutput(str(item["name"]), _artifact_type(item["type"])) + for item in ( + _mapping(value, "artifact output") + for value in raw.get("artifact_outputs", []) + ) + ) + return Node( + id=str(raw["id"]), + kernel=str(raw["kernel"]), + inputs=inputs, + outputs=outputs, + params=params, + population=None if "population" not in raw else str(raw["population"]), + structural=StructuralDelta(str(raw.get("structural", "none"))), + base=None if "base" not in raw else str(raw["base"]), + bases=tuple(str(value) for value in raw.get("bases", [])), + sources=tuple(str(value) for value in raw.get("sources", [])), + weights=weights, + mass=str(raw.get("mass", "conserve")), + description=str(raw.get("description", "")), + citation=str(raw.get("citation", "")), + entrants=bool(raw.get("entrants", False)), + artifact_inputs=artifact_inputs, + artifact_outputs=artifact_outputs, + requires_success=tuple(str(value) for value in raw.get("requires_success", [])), + ) + + +def _lower_product(value: Mapping[str, object]) -> Product: + target = _mapping(value["target"], "product target") + return Product( + name=str(value["name"]), + kind=ProductKind(str(value["kind"])), + node=None if "node" not in target else str(target["node"]), + entity=None if "entity" not in target else str(target["entity"]), + column=None if "column" not in target else str(target["column"]), + artifact=None if "artifact" not in target else str(target["artifact"]), + source=None if "product" not in target else str(target["product"]), + codec=None if "codec" not in value else str(value["codec"]), + codec_version=value.get("codec_version"), # type: ignore[arg-type] + ) + + +def load_graph_source( + path: str | Path, *, parameters: Mapping[str, object] | None = None +) -> LoadedGraphSource: + """Load a root and its exact modules without executing runtime code.""" + root_path = Path(path) + if root_path.is_symlink(): + raise GraphSourceCompositionError( + "root graph path may not be a symbolic link", source=str(root_path) + ) + root_path = root_path.resolve() + root_dir = root_path.parent + documents: list[_Document] = [] + active: list[Path] = [] + seen: set[Path] = set() + + def visit(candidate: Path, *, is_module: bool) -> None: + if candidate in active: + chain = " -> ".join( + item.relative_to(root_dir).as_posix() for item in (*active, candidate) + ) + raise GraphSourceCompositionError( + f"module cycle: {chain}", source=str(candidate) + ) + if candidate in seen: + raise GraphSourceCompositionError( + f"module is declared more than once: {candidate.relative_to(root_dir).as_posix()}", + source=str(candidate), + ) + active.append(candidate) + document = _read_document(candidate, root_dir, module=is_module) + payload = _mapping(document.parsed.value, "graph document") + for module_path in sorted(payload.get("modules", [])): + visit( + _safe_module_path(root_dir, module_path, source=candidate), + is_module=True, + ) + active.pop() + seen.add(candidate) + documents.append(document) + + visit(root_path, is_module=False) + root = _mapping( + next(item.parsed.value for item in documents if item.path == root_path), "root" + ) + collected: dict[str, list[tuple[Path, object]]] = { + "sources": [], + "nodes": [], + "parameters": [], + "products": [], + } + for document in documents: + payload = _mapping(document.parsed.value, "graph document") + for name in collected: + raw = payload.get(name, {} if name == "parameters" else []) + entries = ( + raw.items() + if name == "parameters" + else ( + ( + str( + _mapping(item, name[:-1])[ + "name" if name != "nodes" else "id" + ] + ), + item, + ) + for item in raw + ) + ) + for key, value in entries: + collected[name].append((document.path, (str(key), value))) + + unique: dict[str, dict[str, object]] = {name: {} for name in collected} + origins: dict[str, dict[str, Path]] = {name: {} for name in collected} + for category, entries in collected.items(): + for source, pair in entries: + key, value = pair + if key in unique[category]: + raise GraphSourceCompositionError( + f"duplicate {category[:-1]} {key!r}; first declared in {origins[category][key]}", + source=str(source), + ) + unique[category][key] = value + origins[category][key] = source + + parameter_declarations = { + name: _mapping(value, f"parameter {name}") + for name, value in unique["parameters"].items() + } + bound = _bind_parameters(parameter_declarations, parameters or {}) + sources = tuple( + SourceRef( + name=str(item["name"]), + codec=str(item["codec"]), + description=str(item.get("description", "")), + content_type=str(item.get("content_type", "application/octet-stream")), + access=None if "access" not in item else str(item["access"]), + expected=tuple( + ExpectedContent( + sha256=str(expected["sha256"]), + boundary=str(expected.get("boundary", "source")), + path=None if "path" not in expected else str(expected["path"]), + size=None if "size" not in expected else int(expected["size"]), + identity_ref=str(expected.get("identity_ref", "")), + ) + for expected in ( + _mapping(value, "expected source content") + for value in item.get("expected", []) + ) + ), + ) + for item in ( + _mapping(unique["sources"][name], "source") + for name in sorted(unique["sources"]) + ) + ) + nodes = tuple( + _lower_node(_mapping(unique["nodes"][name], "node"), bound) + for name in sorted(unique["nodes"]) + ) + products = tuple( + _lower_product(_mapping(unique["products"][name], "product")) + for name in sorted(unique["products"]) + ) + mass_partition = root.get("mass_partition") + try: + graph = Graph( + str(root["country"]), + sources, + nodes, + None + if mass_partition is None + else tuple(str(value) for value in mass_partition), # type: ignore[arg-type] + products, + ) + except (GraphError, TypeError, ValueError) as error: + raise GraphSourceValidationError( + f"graph declaration is invalid: {error}", source=str(root_path) + ) from error + receipts = tuple( + GraphSourceReceipt( + document.relative, + hashlib.sha256(document.raw).hexdigest(), + ) + for document in sorted(documents, key=lambda item: item.relative) + ) + return LoadedGraphSource( + graph, + GRAPH_SOURCE_SCHEMA_VERSION, + bound, + receipts, + graph.products, + ) + + +def graph_from_yaml_file( + path: str | Path, *, parameters: Mapping[str, object] | None = None +) -> Graph: + return load_graph_source(path, parameters=parameters).graph + + +def compiled_graph_from_yaml_file( + path: str | Path, + *, + kernels: KernelRegistry, + parameters: Mapping[str, object] | None = None, +) -> CompiledGraph: + compiled = compile_graph(graph_from_yaml_file(path, parameters=parameters)) + validate_kernel_registry(compiled, kernels) + return compiled + + +def validate_kernel_registry(compiled: CompiledGraph, kernels: KernelRegistry) -> None: + """Verify executable contracts for every node before a run starts.""" + + for node_id in compiled.order: + node = compiled.graph.node(node_id) + try: + kernel = kernels.get(node.kernel) + except KeyError as error: + raise GraphSourceValidationError( + f"node {node.id!r} references unregistered kernel {node.kernel!r}" + ) from error + if kernel.capabilities.structural is not node.structural: + raise GraphSourceValidationError( + f"node {node.id!r} declares structural operation " + f"{node.structural.value!r}, but {node.kernel!r} declares " + f"{kernel.capabilities.structural.value!r}" + ) + for binding in node.artifact_inputs: + producer = compiled.graph.node(binding.producer) + producer_scope = numeric_scope(kernels.get(producer.kernel).capabilities) + try: + require_compatible_scope(producer_scope, kernel.capabilities) + except ValueError as error: + raise GraphSourceValidationError( + f"node {node.id!r} cannot consume typed artifact " + f"{binding.name!r}: {error}" + ) from error + for product in compiled.graph.products: + if product.kind is not ProductKind.VALIDATION: + continue + assert product.node is not None + kernel = kernels.get(compiled.graph.node(product.node).kernel) + if kernel.capabilities.role.value != "gate": + raise GraphSourceValidationError( + f"validation product {product.name!r} targets node " + f"{product.node!r}, whose kernel is not a validation kernel" + ) + + +__all__ = [ + "GRAPH_SOURCE_SCHEMA_VERSION", + "GraphSourceReceipt", + "LoadedGraphSource", + "compiled_graph_from_yaml_file", + "graph_from_yaml_file", + "load_graph_source", + "validate_kernel_registry", +] diff --git a/packages/microcosm-graph/src/microcosm/graph/kernel.py b/packages/microcosm-graph/src/microcosm/graph/kernel.py index 0846a3eaa..cdaa25465 100644 --- a/packages/microcosm-graph/src/microcosm/graph/kernel.py +++ b/packages/microcosm-graph/src/microcosm/graph/kernel.py @@ -52,7 +52,7 @@ from importlib import metadata as importlib_metadata from pathlib import Path from types import MappingProxyType, ModuleType -from typing import Protocol, runtime_checkable +from typing import TYPE_CHECKING, Protocol, runtime_checkable import numpy as np import pandas as pd @@ -61,6 +61,9 @@ from .decl import ArtifactType, Node, Param, StructuralDelta +if TYPE_CHECKING: + from .codecs import BoundSource + __all__ = [ "ArtifactValue", "Capabilities", @@ -313,6 +316,8 @@ class KernelContext: responsible for. weights: Entity name to effective typed weights, for entities named in the node's inputs or outputs. + weight_anchors: Named earlier weight products requested by the node's + weight transition, aligned to the transition population. strata: Read-only per-person strata of the population version. params: The node's parameters. rng: The default generator seeded from the node key. KEYED kernels @@ -321,8 +326,9 @@ class KernelContext: artifacts: Immutable typed bytes for declared artifact aliases only. Consumers validate versioned payloads before using them; nominal types do not themselves verify arbitrary serialized data. - sources: Source name to a content-verified path, for declared - sources only. + sources: Source name to a verified path and authoritative decoder, + for declared sources only. ``BoundSource.decode()`` uses the + codec fixed by the graph declaration. tolerances: ``(entity, column)`` of each declared input column to the :class:`Tolerance` its owning kernel declared, or ``None`` for a bitwise owner. A gate compares against these. @@ -339,10 +345,11 @@ class KernelContext: strata: pd.Series params: Mapping[str, Param] rng: np.random.Generator - sources: Mapping[str, Path] = field(default_factory=dict) + sources: Mapping[str, BoundSource] = field(default_factory=dict) tolerances: Mapping[tuple[str, str], Tolerance | None] = field(default_factory=dict) numerics: Mapping[tuple[str, str], NumericScope] = field(default_factory=dict) artifacts: Mapping[str, ArtifactValue] = field(default_factory=dict) + weight_anchors: Mapping[str, Weights] = field(default_factory=dict) def __post_init__(self) -> None: values = dict(self.artifacts) @@ -356,6 +363,13 @@ def __post_init__(self) -> None: "KernelContext.artifacts must map non-empty aliases to ArtifactValue." ) object.__setattr__(self, "artifacts", MappingProxyType(values)) + anchors = dict(self.weight_anchors) + if any( + not isinstance(name, str) or not name or not isinstance(value, Weights) + for name, value in anchors.items() + ): + raise TypeError("KernelContext.weight_anchors must map names to Weights.") + object.__setattr__(self, "weight_anchors", MappingProxyType(anchors)) @dataclass(frozen=True) diff --git a/packages/microcosm-graph/src/microcosm/graph/keys.py b/packages/microcosm-graph/src/microcosm/graph/keys.py index 99bb9f5c5..d2327b750 100644 --- a/packages/microcosm-graph/src/microcosm/graph/keys.py +++ b/packages/microcosm-graph/src/microcosm/graph/keys.py @@ -9,7 +9,7 @@ from pathlib import Path from .canonical import canonical_json, normative, sha256_domain -from .decl import CompiledGraph, StructuralDelta +from .decl import CompiledGraph, Graph, StructuralDelta from .kernel import Capabilities, Numeric __all__ = [ @@ -17,9 +17,14 @@ "platform_fingerprint", "artifact_key", "frame_key", + "graph_key", "node_key", "seed", "source_content_key", + "source_content_identity", + "source_binding_key", + "validation_outcome_key", + "union_lineage_key", "weights_key", ] @@ -59,16 +64,54 @@ def source_content_key(name: str, path: str | Path) -> str: based ``frame-store`` and ``csv-tables`` codecs. """ + _, content_hash, size = source_content_identity(path) + return _hash_parts("source", name, content_hash, size) + + +def source_content_identity(path: str | Path) -> tuple[str, str, int]: + """Return ``(boundary kind, SHA-256, size)`` without including a path.""" + source_path = Path(path) if source_path.is_file(): content = source_path.read_bytes() - content_hash = hashlib.sha256(content).hexdigest() - size = len(content) - elif source_path.is_dir(): + return "file", hashlib.sha256(content).hexdigest(), len(content) + if source_path.is_dir(): content_hash, size = _directory_identity(source_path) - else: - raise FileNotFoundError(f"Source path does not exist: {source_path}") - return _hash_parts("source", name, content_hash, size) + return "directory", content_hash, size + raise FileNotFoundError(f"Source path does not exist: {source_path}") + + +def source_binding_key( + content_key: str, codec: str, codec_impl_hash: str, content_type: str +) -> str: + """Bind verified bytes to their declared decoder and logical content type.""" + + return _hash_parts( + "source-binding", content_key, codec, codec_impl_hash, content_type + ) + + +def graph_key(graph: Graph) -> str: + """Return the semantic identity of a complete Graph, independent of order.""" + + if not isinstance(graph, Graph): + raise TypeError("graph_key requires a Graph declaration.") + payload = { + "country": graph.country, + "mass_partition": graph.mass_partition, + "sources": tuple( + normative(source) + for source in sorted(graph.sources, key=lambda item: item.name) + ), + "nodes": tuple( + normative(node) for node in sorted(graph.nodes, key=lambda item: item.id) + ), + "products": tuple( + normative(product) + for product in sorted(graph.products, key=lambda item: item.name) + ), + } + return _hash_parts("semantic-graph", payload) def platform_fingerprint() -> str: @@ -106,6 +149,18 @@ def weights_key(node_key: str, entity: str) -> str: return _hash_parts("weights", node_key, entity) +def validation_outcome_key(node_key: str) -> str: + """Identity of the executor-preserved validation outcome for one node.""" + + return _hash_parts("validation-outcome", node_key) + + +def union_lineage_key(node_key: str) -> str: + """Identity of deterministic per-row source lineage for a union node.""" + + return _hash_parts("union-lineage", node_key) + + def _required_key(keys: Mapping[str, str], node_id: str, consumer: str) -> str: try: return keys[node_id] @@ -169,7 +224,17 @@ def node_key( elif node.structural is StructuralDelta.NONE: input_version = compiled.versions[node_id] else: - input_version = node.base + input_version = None if node.structural is StructuralDelta.UNION else node.base + + def supplier(version: str, entity: str, column: str) -> str: + owner = compiled.owners.get((version, entity, column)) + if owner is not None: + return owner + holder = compiled.graph.node(version) + if holder.structural is StructuralDelta.REVISION: + assert holder.base is not None + return supplier(holder.base, entity, column) + return version resolved: dict[tuple[str, str], str] = {} rewritten = { @@ -182,9 +247,7 @@ def node_key( producer = ( input_version if coordinate in rewritten - else compiled.owners.get( - (input_version, slice_.entity, column), input_version - ) + else supplier(input_version, slice_.entity, column) ) producer_key = _required_key(input_keys, producer, node_id) resolved[coordinate] = artifact_key(producer_key, slice_.entity, column) @@ -200,6 +263,21 @@ def node_key( } elif node.structural is StructuralDelta.CREATE: population_input = {} + elif node.structural is StructuralDelta.UNION: + population_input = { + "bases": tuple( + ( + base, + frame_key(_required_key(input_keys, base, node_id)), + tuple( + (member, _required_key(input_keys, member, node_id)) + for member in compiled.predecessors[node_id] + if compiled.versions.get(member) == base and member != base + ), + ) + for base in node.bases + ) + } else: assert node.base is not None population_input = { @@ -267,6 +345,25 @@ def node_key( if node.artifact_inputs else () ) + required_outcomes = ( + ( + { + "required_validation_outcomes": tuple( + ( + product_name, + _required_key( + input_keys, + compiled.product_nodes[product_name], + node_id, + ), + ) + for product_name in sorted(node.requires_success) + ) + }, + ) + if node.requires_success + else () + ) return _hash_parts( "node", normative(node), @@ -278,6 +375,7 @@ def node_key( capabilities, *platform_scope, *typed_inputs, + *required_outcomes, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/manifest.py b/packages/microcosm-graph/src/microcosm/graph/manifest.py index 34b121a78..922e33109 100644 --- a/packages/microcosm-graph/src/microcosm/graph/manifest.py +++ b/packages/microcosm-graph/src/microcosm/graph/manifest.py @@ -36,6 +36,7 @@ _SCHEMA_VERSION = 2 _TYPED_SCHEMA_VERSION = 3 +_GRAPH_BOUND_SCHEMA_VERSION = 4 _LEGACY_SCHEMA_VERSION = 1 _CERTIFYING_GATE_OUTCOMES = frozenset({"pass", "not_applicable"}) @@ -223,6 +224,9 @@ class NodeReceipt: opaque_artifacts: Mapping[str, str] = field(default_factory=dict) legacy_capabilities: bool = field(default=False, kw_only=True) typed_artifacts: Mapping[str, object] = field(default_factory=dict, kw_only=True) + status: str = field(default="executed", kw_only=True) + blocked_by: tuple[str, ...] = field(default=(), kw_only=True) + outcome_key: str | None = field(default=None, kw_only=True) def __post_init__(self) -> None: if not isinstance(self.key, str): @@ -237,6 +241,20 @@ def __post_init__(self) -> None: raise TypeError("NodeReceipt.kernel_impl_hash must be a string") if not isinstance(self.legacy_capabilities, bool): raise TypeError("NodeReceipt.legacy_capabilities must be a bool") + if self.status not in {"executed", "unreached"}: + raise ValueError("NodeReceipt.status must be 'executed' or 'unreached'.") + if not isinstance(self.blocked_by, tuple) or any( + not isinstance(node_id, str) or not node_id for node_id in self.blocked_by + ): + raise TypeError("NodeReceipt.blocked_by must contain node ids.") + if len(set(self.blocked_by)) != len(self.blocked_by): + raise ValueError("NodeReceipt.blocked_by contains duplicates.") + if self.status == "executed" and self.blocked_by: + raise ValueError("An executed NodeReceipt cannot be blocked.") + if self.status == "unreached" and not self.blocked_by: + raise ValueError("An unreached NodeReceipt must name what blocked it.") + if self.outcome_key is not None and not isinstance(self.outcome_key, str): + raise TypeError("NodeReceipt.outcome_key must be a string or None.") if self.legacy_capabilities: if not isinstance(self.capabilities, Mapping) or isinstance( self.capabilities, Capabilities @@ -370,6 +388,9 @@ def _payload(self) -> dict[str, object]: "frame_key": self.frame_key, "weight_key": self.weight_key, "opaque_artifacts": self.opaque_artifacts, + **({"status": self.status} if self.status != "executed" else {}), + **({"blocked_by": self.blocked_by} if self.blocked_by else {}), + **({"outcome_key": self.outcome_key} if self.outcome_key else {}), **( {"typed_artifacts": self.typed_artifacts} if self.typed_artifacts @@ -435,6 +456,12 @@ class RunManifest: mass_ledgers: Mapping[str, tuple[MassRecord, ...]] = field( default_factory=dict, repr=False, compare=False ) + graph_key: str = "" + graph_source_receipts: tuple[Mapping[str, object], ...] = () + parameters: Mapping[str, object] = field(default_factory=dict) + source_bindings: Mapping[str, object] = field(default_factory=dict) + graph_json_key: str | None = None + products: Mapping[str, object] = field(default_factory=dict) def __post_init__(self) -> None: if not isinstance(self.country, str): @@ -485,6 +512,38 @@ def __post_init__(self) -> None: ) mass_ledgers[version_id] = frozen_records object.__setattr__(self, "mass_ledgers", MappingProxyType(mass_ledgers)) + if self.graph_key and ( + len(self.graph_key) != 64 + or any(character not in "0123456789abcdef" for character in self.graph_key) + ): + raise ValueError("RunManifest.graph_key must be a SHA-256 identity.") + if self.graph_json_key is not None and ( + not isinstance(self.graph_json_key, str) + or len(self.graph_json_key) != 64 + or any( + character not in "0123456789abcdef" for character in self.graph_json_key + ) + ): + raise ValueError("RunManifest.graph_json_key must be a SHA-256 identity.") + graph_receipts = _freeze_json(self.graph_source_receipts) + if not isinstance(graph_receipts, tuple) or any( + not isinstance(receipt, Mapping) for receipt in graph_receipts + ): + raise TypeError("RunManifest.graph_source_receipts must contain mappings.") + object.__setattr__(self, "graph_source_receipts", graph_receipts) + for field_name in ("parameters", "source_bindings", "products"): + frozen = _freeze_json(getattr(self, field_name)) + if not isinstance(frozen, Mapping): + raise TypeError(f"RunManifest.{field_name} must be a mapping.") + object.__setattr__(self, field_name, frozen) + if not self.graph_key and ( + self.graph_source_receipts + or self.parameters + or self.source_bindings + or self.graph_json_key is not None + or self.products + ): + raise ValueError("Graph-bound manifest fields require graph_key.") @property def content_addressed(self) -> Mapping[str, object]: @@ -494,11 +553,32 @@ def content_addressed(self) -> Mapping[str, object]: node_id: self.nodes[node_id]._content_payload() for node_id in sorted(self.nodes) } - return MappingProxyType( - { - "nodes": MappingProxyType(nodes), - "tier": self.tier, - } + payload: dict[str, object] = { + "nodes": MappingProxyType(nodes), + "tier": self.tier, + } + if self.graph_key: + payload.update( + { + "graph_key": self.graph_key, + "graph_source_receipts": self.graph_source_receipts, + "parameters": self.parameters, + "source_bindings": self.source_bindings, + "graph_json_key": self.graph_json_key, + "products": self.products, + "outcome": self.outcome, + } + ) + return MappingProxyType(payload) + + @property + def outcome(self) -> str: + """Run-level result derived from required validation dependencies.""" + + return ( + "not_successful" + if any(node.status == "unreached" for node in self.nodes.values()) + else "success" ) @property @@ -515,6 +595,8 @@ def tier(self) -> str | None: if len(releases) != 1: raise ValueError("a run manifest must contain at most one release node") node_id, release = releases[0] + if release.status == "unreached": + return None gate_ancestry = release.receipt.get("gate_ancestry") if not isinstance(gate_ancestry, tuple) or any( not isinstance(gate_id, str) or not gate_id for gate_id in gate_ancestry @@ -621,9 +703,13 @@ def to_json(self) -> str: """Serialize the complete portable provenance as canonical JSON.""" payload = { - "schema_version": _TYPED_SCHEMA_VERSION - if any(node.typed_artifacts for node in self.nodes.values()) - else _SCHEMA_VERSION, + "schema_version": ( + _GRAPH_BOUND_SCHEMA_VERSION + if self.graph_key + else _TYPED_SCHEMA_VERSION + if any(node.typed_artifacts for node in self.nodes.values()) + else _SCHEMA_VERSION + ), "key": self.key, "tier": self.tier, "known_failures": self.known_failures, @@ -636,6 +722,19 @@ def to_json(self) -> str: "started_at": self.started_at, "finished_at": self.finished_at, "host": self.host, + **( + { + "graph_key": self.graph_key, + "graph_source_receipts": self.graph_source_receipts, + "parameters": self.parameters, + "source_bindings": self.source_bindings, + "graph_json_key": self.graph_json_key, + "products": self.products, + "outcome": self.outcome, + } + if self.graph_key + else {} + ), } return canonical_json(payload).decode("utf-8") @@ -665,6 +764,7 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: _LEGACY_SCHEMA_VERSION, _SCHEMA_VERSION, _TYPED_SCHEMA_VERSION, + _GRAPH_BOUND_SCHEMA_VERSION, }: raise ValueError(f"unsupported manifest schema version {schema_version!r}") @@ -688,11 +788,44 @@ def from_json(cls, value: str | bytes | bytearray) -> Self: started_at=_string_field(raw, "started_at"), finished_at=_string_field(raw, "finished_at"), host=_string_field(raw, "host"), + graph_key=( + _string_field(raw, "graph_key") + if schema_version == _GRAPH_BOUND_SCHEMA_VERSION + else "" + ), + graph_source_receipts=tuple( + raw.get("graph_source_receipts", ()) + if schema_version == _GRAPH_BOUND_SCHEMA_VERSION + else () + ), + parameters=( + raw.get("parameters", {}) + if schema_version == _GRAPH_BOUND_SCHEMA_VERSION + else {} + ), + source_bindings=( + raw.get("source_bindings", {}) + if schema_version == _GRAPH_BOUND_SCHEMA_VERSION + else {} + ), + graph_json_key=( + raw.get("graph_json_key") + if schema_version == _GRAPH_BOUND_SCHEMA_VERSION + else None + ), + products=( + raw.get("products", {}) + if schema_version == _GRAPH_BOUND_SCHEMA_VERSION + else {} + ), ) if schema_version == _TYPED_SCHEMA_VERSION and not any( node.typed_artifacts for node in nodes.values() ): raise ValueError("Schema-v3 manifest must carry typed artifact provenance.") + if schema_version == _GRAPH_BOUND_SCHEMA_VERSION: + if raw.get("outcome") != manifest.outcome: + raise ValueError("manifest run outcome differs from node receipts") body = raw.get("content_addressed") if not isinstance(body, Mapping): raise ValueError("manifest content-addressed body must be an object") @@ -776,6 +909,18 @@ def load(cls, path: str | Path, store: ContentStore) -> Self: "finished_at", "host", } + if raw.get("schema_version") == _GRAPH_BOUND_SCHEMA_VERSION: + required.update( + { + "graph_key", + "graph_source_receipts", + "parameters", + "source_bindings", + "graph_json_key", + "products", + "outcome", + } + ) missing = sorted(required - set(raw)) if missing: raise StoreCorruptError( @@ -885,7 +1030,7 @@ def _validate_current_content_addressed_body( continue raise ValueError(f"manifest content key mismatch at node {node_id!r}: {detail}") - expected_fields = {"nodes", "tier"} + expected_fields = set(expected) if set(body) != expected_fields: raise ValueError( f"manifest content key mismatch after node {first_node!r}: body fields " @@ -903,6 +1048,15 @@ def _validate_current_content_addressed_body( f"{body.get('tier')!r} differs from derived tier " f"{expected.get('tier')!r}" ) + for field_name in sorted(expected_fields - {"nodes", "tier"}): + if canonical_json(body.get(field_name)) != canonical_json( + expected.get(field_name) + ): + raise ValueError( + "manifest content key mismatch after node " + f"{first_node!r}: field {field_name!r} differs from portable " + "provenance" + ) def _capability_role(node: NodeReceipt) -> KernelRole: @@ -984,8 +1138,17 @@ def _validate_artifacts(manifest: RunManifest, store: ContentStore) -> None: store.metadata(node.frame_key, kind="frame") if node.weight_key is not None: store.metadata(node.weight_key, kind="column") + if node.outcome_key is not None: + store.metadata(node.outcome_key, kind="validation-outcome") + union_lineage = node.receipt.get("union_lineage") + if isinstance(union_lineage, Mapping) and isinstance( + union_lineage.get("key"), str + ): + store.metadata(union_lineage["key"], kind="union-lineage") for key in node.opaque_artifacts.values(): store.metadata(key, kind="bytes") + if manifest.graph_json_key is not None: + store.metadata(manifest.graph_json_key, kind="bytes") def _string_field(payload: Mapping[str, object], name: str) -> str: @@ -1127,8 +1290,14 @@ def _node_receipt_from_payload(value: object, *, schema_version: int) -> NodeRec weight_key = value.get("weight_key") opaque_artifacts = value.get("opaque_artifacts", {}) typed_artifacts = value.get("typed_artifacts", {}) - if "typed_artifacts" in value and schema_version != _TYPED_SCHEMA_VERSION: - raise ValueError("Typed artifact provenance requires manifest schema 3.") + status = value.get("status", "executed") + blocked_by = value.get("blocked_by", ()) + outcome_key = value.get("outcome_key") + if "typed_artifacts" in value and schema_version not in { + _TYPED_SCHEMA_VERSION, + _GRAPH_BOUND_SCHEMA_VERSION, + }: + raise ValueError("Typed artifact provenance requires manifest schema 3 or 4.") capabilities_payload = value.get("capabilities") if schema_version == _LEGACY_SCHEMA_VERSION: # Every schema-v1 receipt is legacy: v1 never recorded a tolerance, so @@ -1177,6 +1346,12 @@ def _node_receipt_from_payload(value: object, *, schema_version: int) -> NodeRec for name, key in opaque_artifacts.items() ): raise ValueError("node receipt opaque_artifacts must map strings to strings") + if not isinstance(status, str): + raise ValueError("node receipt status must be a string") + if not isinstance(blocked_by, list | tuple): + raise ValueError("node receipt blocked_by must be an array") + if outcome_key is not None and not isinstance(outcome_key, str): + raise ValueError("node receipt outcome_key must be a string or null") artifacts: dict[tuple[str, str], str] = {} for item in artifacts_raw: if not isinstance(item, Mapping): @@ -1203,6 +1378,9 @@ def _node_receipt_from_payload(value: object, *, schema_version: int) -> NodeRec opaque_artifacts=opaque_artifacts, legacy_capabilities=legacy_capabilities, typed_artifacts=typed_artifacts, + status=status, + blocked_by=tuple(str(node_id) for node_id in blocked_by), + outcome_key=outcome_key, ) diff --git a/packages/microcosm-graph/src/microcosm/graph/materialize.py b/packages/microcosm-graph/src/microcosm/graph/materialize.py new file mode 100644 index 000000000..6bc17a143 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/materialize.py @@ -0,0 +1,306 @@ +"""Deterministic post-run materialization of declared local export products.""" + +from __future__ import annotations + +import os +import re +import uuid +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from types import MappingProxyType +from typing import Protocol + +from .canonical import canonical_json, sha256_domain +from .decl import Graph, Product, ProductKind, compile_graph +from .errors import GraphRuntimeError +from .keys import graph_key, source_content_identity +from .manifest import RunManifest +from .reconstruct import reconstruct_population +from .store import ContentStore + +__all__ = [ + "CandidateIndex", + "Materializer", + "MaterializedProduct", + "MaterializerRegistry", + "materialize_products", +] + +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") + + +class Materializer(Protocol): + """A deterministic writer for one declared product codec and version.""" + + def __call__(self, value: object, destination: Path) -> None: + """Write ``value`` below the caller-provided local destination.""" + ... + + +@dataclass(frozen=True) +class _RegisteredMaterializer: + writer: Materializer + implementation_hash: str + + +class MaterializerRegistry: + """Exact codec/version bindings used only after graph execution.""" + + def __init__(self) -> None: + self._materializers: dict[tuple[str, int], _RegisteredMaterializer] = {} + + def register( + self, + codec: str, + version: int, + writer: Materializer, + *, + implementation_hash: str, + ) -> None: + if not isinstance(codec, str) or not codec: + raise TypeError("Materializer codec must be a non-empty string.") + if type(version) is not int or version < 1: + raise TypeError("Materializer version must be a positive integer.") + if not callable(writer): + raise TypeError("Materializer writer must be callable.") + if not isinstance(implementation_hash, str) or not _SHA256.fullmatch( + implementation_hash + ): + raise TypeError("Materializer implementation_hash must be SHA-256.") + key = (codec, version) + if key in self._materializers: + raise ValueError(f"Materializer {codec!r} version {version} is registered.") + self._materializers[key] = _RegisteredMaterializer(writer, implementation_hash) + + def get(self, codec: str, version: int) -> _RegisteredMaterializer: + try: + return self._materializers[(codec, version)] + except KeyError as error: + raise GraphRuntimeError( + f"No materializer is registered for {codec!r} version {version}." + ) from error + + +@dataclass(frozen=True) +class MaterializedProduct: + """Portable identity record for one local compatibility artifact.""" + + name: str + source: str + codec: str + codec_version: int + codec_impl_hash: str + path: str + boundary: str + sha256: str + size: int + content_key: str + + def payload(self) -> Mapping[str, object]: + return MappingProxyType( + { + "name": self.name, + "source": self.source, + "codec": self.codec, + "codec_version": self.codec_version, + "codec_impl_hash": self.codec_impl_hash, + "path": self.path, + "boundary": self.boundary, + "sha256": self.sha256, + "size": self.size, + "content_key": self.content_key, + } + ) + + +@dataclass(frozen=True) +class CandidateIndex: + """The complete local result of post-run materialization.""" + + manifest_key: str + graph_key: str + products: Mapping[str, MaterializedProduct] + + def __post_init__(self) -> None: + object.__setattr__(self, "products", MappingProxyType(dict(self.products))) + + @property + def key(self) -> str: + return sha256_domain("candidate-index", canonical_json(self.content)) + + @property + def content(self) -> Mapping[str, object]: + return MappingProxyType( + { + "schema_version": 1, + "manifest_key": self.manifest_key, + "graph_key": self.graph_key, + "products": { + name: self.products[name].payload() + for name in sorted(self.products) + }, + } + ) + + def to_json(self) -> str: + return canonical_json({**self.content, "key": self.key}).decode("utf-8") + + +def _relative_output(value: str) -> PurePosixPath: + path = PurePosixPath(value) + if not value or path.is_absolute() or ".." in path.parts or "." in path.parts: + raise ValueError( + f"Materialized product path {value!r} is not a safe relative path." + ) + if value == "candidate-index.json": + raise ValueError("candidate-index.json is reserved for the candidate index.") + return path + + +def _stored_product( + graph: Graph, + manifest: RunManifest, + store: ContentStore, + product: Product, +) -> object: + record = manifest.products.get(product.name) + if not isinstance(record, Mapping) or record.get("status") != "executed": + raise GraphRuntimeError( + f"Stored product {product.name!r} was not successfully produced." + ) + key = record.get("key") + if product.kind is ProductKind.POPULATION: + return reconstruct_population(graph, manifest, store, product.name) + if not isinstance(key, str): + raise GraphRuntimeError(f"Stored product {product.name!r} has no content key.") + if product.kind is ProductKind.COORDINATE: + producer = record.get("supplier") + if not isinstance(producer, str): + raise GraphRuntimeError( + f"Stored coordinate product {product.name!r} has no supplier." + ) + return store.load_column(key, node_key=manifest.nodes[producer].key) + if product.kind is ProductKind.WEIGHTS: + state = record.get("state") + if not isinstance(state, str): + raise GraphRuntimeError( + f"Stored weights product {product.name!r} has no population state." + ) + if record.get("storage") == "column": + return store.load_column(key, node_key=manifest.nodes[state].key) + if record.get("storage") == "frame": + assert product.entity is not None + return store.load_frame( + key, node_key=manifest.nodes[state].key + ).weights_for(product.entity) + raise GraphRuntimeError( + f"Stored weights product {product.name!r} has unknown storage." + ) + if product.kind is ProductKind.ARTIFACT: + return store.load_bytes(key) + if product.kind is ProductKind.VALIDATION: + return store.load_json(key, kind="validation-outcome") + raise GraphRuntimeError(f"Product {product.name!r} cannot be materialized.") + + +def materialize_products( + graph: Graph, + manifest_path: str | Path, + store: ContentStore, + destination: str | Path, + registry: MaterializerRegistry, + outputs: Mapping[str, str], +) -> CandidateIndex: + """Write requested exports after loading a complete saved run manifest. + + The function writes only below ``destination`` and never performs remote + publication. The candidate index is written last, so its presence denotes + a complete set of requested local outputs. + """ + + compile_graph(graph) + saved_manifest = RunManifest.from_json(Path(manifest_path).read_text("utf-8")) + if saved_manifest.graph_key != graph_key(graph): + raise GraphRuntimeError( + "The saved manifest was produced from a different Graph declaration." + ) + if saved_manifest.outcome != "success": + raise GraphRuntimeError("An unsuccessful run cannot be materialized.") + declarations = {product.name: product for product in graph.products} + root = Path(destination).absolute() + if root.is_symlink(): + raise ValueError(f"Candidate directory may not be a symlink: {root}") + root.mkdir(parents=True, exist_ok=True) + index_path = root / "candidate-index.json" + if index_path.exists(): + raise FileExistsError(f"Candidate index already exists: {index_path}") + + materialized: dict[str, MaterializedProduct] = {} + for name in sorted(outputs): + product = declarations.get(name) + if product is None or product.kind is not ProductKind.EXPORT: + raise GraphRuntimeError(f"Requested product {name!r} is not an export.") + assert product.source is not None + assert product.codec is not None + assert product.codec_version is not None + source = declarations[product.source] + relative = _relative_output(outputs[name]) + output_path = root.joinpath(*relative.parts) + if output_path.exists() or output_path.is_symlink(): + raise FileExistsError(f"Materialized output already exists: {output_path}") + current = root + for part in relative.parts[:-1]: + current /= part + if current.is_symlink(): + raise ValueError( + f"Materialized output traverses a symlink: {output_path}" + ) + current.mkdir(exist_ok=True) + binding = registry.get(product.codec, product.codec_version) + value = _stored_product(graph, saved_manifest, store, source) + binding.writer(value, output_path) + if not output_path.exists() or output_path.is_symlink(): + raise GraphRuntimeError( + f"Materializer for {name!r} did not create the requested path." + ) + if output_path.is_dir() and any( + candidate.is_symlink() for candidate in output_path.rglob("*") + ): + raise GraphRuntimeError( + f"Materializer for {name!r} created a symbolic link." + ) + boundary, digest, size = source_content_identity(output_path) + content_key = sha256_domain( + "materialized-product", + canonical_json( + { + "name": name, + "source": product.source, + "codec": product.codec, + "codec_version": product.codec_version, + "codec_impl_hash": binding.implementation_hash, + "boundary": boundary, + "sha256": digest, + "size": size, + } + ), + ) + materialized[name] = MaterializedProduct( + name=name, + source=product.source, + codec=product.codec, + codec_version=product.codec_version, + codec_impl_hash=binding.implementation_hash, + path=relative.as_posix(), + boundary=boundary, + sha256=digest, + size=size, + content_key=content_key, + ) + + index = CandidateIndex(saved_manifest.key, saved_manifest.graph_key, materialized) + temporary = root / f".candidate-index.{uuid.uuid4().hex}.tmp" + temporary.write_text(index.to_json(), encoding="utf-8") + os.replace(temporary, index_path) + return index diff --git a/packages/microcosm-graph/src/microcosm/graph/population.py b/packages/microcosm-graph/src/microcosm/graph/population.py index 1a52baf3f..967ebc337 100644 --- a/packages/microcosm-graph/src/microcosm/graph/population.py +++ b/packages/microcosm-graph/src/microcosm/graph/population.py @@ -44,6 +44,7 @@ "restore_cached_expand", "storage_equal", "token_for_dtype", + "union_populations", "weight_cap_receipt", ] @@ -362,6 +363,240 @@ def population_from_frame( ) +def _json_id(value: object) -> object: + return value.item() if isinstance(value, np.generic) else value + + +def union_populations( + populations: Mapping[str, Population], node: Node +) -> tuple[Population, Mapping[str, tuple[tuple[object, str, object], ...]]]: + """Combine compatible bases with deterministic ID remapping and lineage.""" + + if node.structural is not StructuralDelta.UNION: + raise PopulationError("union_populations requires a UNION node.") + if set(populations) != set(node.bases): + raise PopulationError( + f"UNION node {node.id!r} needs bases {node.bases!r}; got " + f"{tuple(sorted(populations))!r}." + ) + ordered = [populations[name] for name in node.bases] + first = ordered[0].frame + for name, population in zip(node.bases[1:], ordered[1:], strict=True): + frame = population.frame + if frame.schema != first.schema: + raise PopulationError( + f"UNION node {node.id!r} base {name!r} has a different schema." + ) + if frame.metadata != first.metadata: + raise PopulationError( + f"UNION node {node.id!r} base {name!r} has different metadata." + ) + if frame.links != first.links: + raise PopulationError( + f"UNION node {node.id!r} base {name!r} has different link tables." + ) + if frame.weighted_entities != first.weighted_entities: + raise PopulationError( + f"UNION node {node.id!r} base {name!r} has different weighted entities." + ) + for entity in first.entities: + left = first.table(entity) + right = frame.table(entity) + if tuple(left.columns) != tuple(right.columns): + raise PopulationError( + f"UNION node {node.id!r} base {name!r} has different columns " + f"on {entity!r}." + ) + if tuple(map(str, left.dtypes)) != tuple(map(str, right.dtypes)): + raise PopulationError( + f"UNION node {node.id!r} base {name!r} has different dtypes " + f"on {entity!r}." + ) + for entity in first.weighted_entities: + if frame.weights_for(entity).kind is not first.weights_for(entity).kind: + raise PopulationError( + f"UNION node {node.id!r} base {name!r} has incompatible " + f"{entity!r} weight kind." + ) + + table_parts: dict[str, list[pd.DataFrame]] = { + entity: [] for entity in first.entities + } + link_parts: dict[str, list[pd.DataFrame]] = {name: [] for name in first.links} + weight_parts: dict[str, list[np.ndarray]] = { + entity: [] for entity in first.weighted_entities + } + design_parts: dict[str, list[np.ndarray]] = { + entity: [] + for entity in first.weighted_entities + if all(entity in population.design_weights for population in ordered) + } + used_ids: dict[str, set[object]] = {entity: set() for entity in first.entities} + lineage: dict[str, list[tuple[object, str, object]]] = { + entity: [] for entity in first.entities + } + strata_parts: list[pd.Series] = [] + schema = first.schema + + for base_name, population in zip(node.bases, ordered, strict=True): + frame = population.frame + copied = { + entity: frame.table(entity).copy(deep=True) for entity in frame.entities + } + id_maps: dict[str, dict[object, object]] = {} + for entity in frame.entities: + id_column = schema.entity_id_column(entity) + original = copied[entity][id_column].to_numpy(copy=True) + if len(set(original.tolist())) != len(original): + raise PopulationError( + f"UNION node {node.id!r} base {base_name!r} repeats {entity!r} ids." + ) + overlap = used_ids[entity].intersection(original.tolist()) + remapped = original.copy() + if overlap: + if not np.issubdtype(original.dtype, np.integer) or any( + not isinstance(value, int | np.integer) + for value in used_ids[entity] + ): + raise PopulationError( + f"UNION node {node.id!r} cannot remap colliding non-integer " + f"{entity!r} ids." + ) + offset = int(max(used_ids[entity])) + 1 - int(original.min()) + remapped = original + offset + mapping = { + _json_id(old): _json_id(new) + for old, new in zip(original, remapped, strict=True) + } + id_maps[entity] = mapping + copied[entity][id_column] = remapped + used_ids[entity].update(map(_json_id, remapped)) + lineage[entity].extend( + (_json_id(new), base_name, _json_id(old)) + for old, new in zip(original, remapped, strict=True) + ) + + person = schema.person_entity + for group in schema.group_entities: + membership = schema.membership_column(group) + copied[person][membership] = copied[person][membership].map(id_maps[group]) + for link in schema.links: + table = frame.link(link.name).copy(deep=True) + for entity in (link.left_entity, link.right_entity): + id_column = schema.entity_id_column(entity) + table[id_column] = table[id_column].map(id_maps[entity]) + link_parts[link.name].append(table) + for entity in frame.entities: + table_parts[entity].append(copied[entity]) + if entity in weight_parts: + weight_parts[entity].append(frame.weights_for(entity).values) + if entity in design_parts: + design_parts[entity].append(population.design_weights[entity]) + strata_parts.append(frame.strata.reset_index(drop=True)) + + person = schema.person_entity + tables: dict[str, pd.DataFrame] = { + person: pd.concat(table_parts[person], ignore_index=True) + } + weights: dict[str, Weights] = {} + entity_orders: dict[str, np.ndarray] = { + person: np.arange(len(tables[person]), dtype=np.int64) + } + for group in schema.group_entities: + combined = pd.concat(table_parts[group], ignore_index=True) + order = np.argsort( + combined[schema.entity_id_column(group)].to_numpy(), kind="stable" + ) + entity_orders[group] = order + tables[group] = combined.iloc[order].reset_index(drop=True) + for entity, parts in weight_parts.items(): + values = np.concatenate(parts)[entity_orders[entity]] + weights[entity] = Weights(values, first.weights_for(entity).kind) + for name, parts in link_parts.items(): + tables[name] = pd.concat(parts, ignore_index=True) + frame = Frame( + tables, + schema, + weights, + pd.concat(strata_parts, ignore_index=True), + mass_log=tuple( + record for population in ordered for record in population.frame.mass_log + ), + metadata=first.metadata, + ) + + before_total = sum( + float(population.frame.stratum_mass().sum()) for population in ordered + ) + after_mass = frame.stratum_mass() + after_total = float(after_mass.sum()) + if node.mass == "conserve" and not np.isclose( + before_total, after_total, rtol=_MASS_RTOL, atol=0.0 + ): + raise PopulationError( + f"UNION node {node.id!r} did not preserve the sum of source mass." + ) + if node.mass == "declared": + target = node.params.get("union_target_mass") + if isinstance(target, bool) or not isinstance(target, int | float): + raise PopulationError( + f"UNION node {node.id!r} mass='declared' requires union_target_mass." + ) + if not np.isclose(after_total, float(target), rtol=_MASS_RTOL, atol=0.0): + raise PopulationError( + f"UNION node {node.id!r} produced mass {after_total!r}, not " + f"declared mass {float(target)!r}." + ) + allocation = node.params.get("source_mass_fractions") + if allocation is not None: + if not isinstance(allocation, Mapping) or set(allocation) != set(node.bases): + raise PopulationError( + f"UNION node {node.id!r} source_mass_fractions must name every base." + ) + for base_name, population in zip(node.bases, ordered, strict=True): + expected = allocation[base_name] + actual = float(population.frame.stratum_mass().sum()) / after_total + if ( + isinstance(expected, bool) + or not isinstance(expected, int | float) + or not np.isclose(actual, float(expected), rtol=_MASS_RTOL, atol=0.0) + ): + raise PopulationError( + f"UNION node {node.id!r} source {base_name!r} mass fraction " + f"is {actual!r}, not {expected!r}." + ) + + before_by: dict[object, float] = {} + for population in ordered: + for label, value in population.frame.stratum_mass().items(): + before_by[label] = before_by.get(label, 0.0) + float(value) + record = MassRecord( + node.id, + StructuralDelta.UNION.value, + node.mass, + before_total, + after_total, + tuple(before_by.items()), + tuple((label, float(value)) for label, value in after_mass.items()), + ) + design = { + entity: np.concatenate(parts)[entity_orders[entity]] + for entity, parts in design_parts.items() + } + result = Population.from_frame( + frame, + node.id, + mass_ledger=tuple( + record for population in ordered for record in population.mass_ledger + ) + + (record,), + design_weights=design, + ) + return result, MappingProxyType( + {entity: tuple(entries) for entity, entries in lineage.items()} + ) + + def _lineage_json_scalar( value: object, *, allow_null: bool = False ) -> str | int | float | bool | None: @@ -1056,6 +1291,7 @@ def patch( *, mass_partition: tuple[str, str] | None = None, rewrite_coordinates: frozenset[tuple[str, str]] = frozenset(), + weight_anchor: Weights | None = None, ) -> Population: """Validate and apply one node result without mutating ``population``. @@ -1093,9 +1329,19 @@ def patch( ) before = population.frame - if node.structural is StructuralDelta.NONE: + if node.weights is None or node.weights.anchor is None: + if weight_anchor is not None: + raise PopulationError( + f"Node {node.id!r} received an undeclared weight-state anchor." + ) + elif weight_anchor is None: + raise PopulationError( + f"Node {node.id!r} did not receive declared weight-state anchor " + f"{node.weights.anchor!r}." + ) + if node.structural in {StructuralDelta.NONE, StructuralDelta.REVISION}: if result.frame is not None: - raise PopulationError(f"Non-structural node {node.id!r} returned a Frame.") + raise PopulationError(f"Value-producing node {node.id!r} returned a Frame.") frame, owners = _patch_columns(population, node, result) elif lineage_expand: frame, owners = _patch_expand( @@ -1134,10 +1380,29 @@ def patch( _assert_design_weight_cap(frame, design_weights, node) ledger = population.mass_ledger - if node.structural is not StructuralDelta.NONE or node.weights is not None: + if ( + node.structural + not in { + StructuralDelta.NONE, + StructuralDelta.REVISION, + } + or node.weights is not None + ): policy = _mass_policy(node) + mass_before = before + if weight_anchor is not None: + assert node.weights is not None + try: + mass_before = _replace_weights( + before, node.weights.entity, weight_anchor + ) + except (TypeError, ValueError) as error: + raise PopulationError( + f"Node {node.id!r} weight-state anchor is not aligned to " + f"{node.weights.entity!r}." + ) from error record = _mass_record( - before, + mass_before, frame, node, result, @@ -2200,6 +2465,8 @@ def _design_cap(node: Node) -> tuple[str, float] | None: transition = node.weights if transition is None or transition.to_kind != WeightKind.CALIBRATED.value: return None + if transition.anchor is not None: + return None raw_cap = node.params.get("max_weight_ratio") if raw_cap is None: return None diff --git a/packages/microcosm-graph/src/microcosm/graph/reconstruct.py b/packages/microcosm-graph/src/microcosm/graph/reconstruct.py new file mode 100644 index 000000000..bc91b076a --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/reconstruct.py @@ -0,0 +1,148 @@ +"""Portable reconstruction of named population products from stored values.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from microcosm.frame import Frame + +from .decl import Graph, ProductKind, StructuralDelta, compile_graph +from .errors import GraphRuntimeError, StoreCorruptError +from .kernel import KernelResult +from .keys import graph_key +from .manifest import RunManifest +from .population import Population, patch +from .store import ContentStore + +__all__ = ["reconstruct_population"] + + +def _owners_for_stored_frame( + frame: Frame, + *, + node_id: str, + base: Population | None, +) -> Mapping[tuple[str, str], str]: + """Restore the ownership needed to apply later stored column patches.""" + + coordinates = { + (entity, str(column)) + for entity in frame.entities + for column in frame.table(entity).columns + } + if base is not None and set(base.owners) == coordinates: + return base.owners + return {coordinate: node_id for coordinate in coordinates} + + +def reconstruct_population( + graph: Graph, + manifest: RunManifest, + store: ContentStore, + product_name: str, +) -> Frame: + """Reconstruct a named population without kernels or original source files. + + Structural states are restored from their stored frames. Ordinary and + revision nodes are replayed as stored column patches in canonical execution + order through the product's declared node. + """ + + if manifest.graph_key != graph_key(graph): + raise GraphRuntimeError( + "The manifest was produced from a different Graph declaration." + ) + compiled = compile_graph(graph) + if set(manifest.nodes) != set(compiled.order): + raise GraphRuntimeError( + "The manifest does not contain exactly the compiled Graph nodes." + ) + declarations = {product.name: product for product in graph.products} + try: + product = declarations[product_name] + except KeyError as error: + raise KeyError(f"Graph has no product named {product_name!r}.") from error + if product.kind is not ProductKind.POPULATION: + raise GraphRuntimeError( + f"Product {product_name!r} is {product.kind.value!r}, not a population." + ) + assert product.node is not None + try: + target_index = compiled.order.index(product.node) + except ValueError as error: # defended by compilation + raise GraphRuntimeError( + f"Population product {product_name!r} names an unknown node." + ) from error + + product_record = manifest.products.get(product_name) + if not isinstance(product_record, Mapping): + raise GraphRuntimeError( + f"Manifest has no record for population product {product_name!r}." + ) + if ( + product_record.get("kind") != ProductKind.POPULATION.value + or product_record.get("producer") != product.node + or product_record.get("status") != "executed" + ): + raise GraphRuntimeError( + f"Manifest record for population product {product_name!r} is unusable." + ) + + populations: dict[str, Population] = {} + for node_id in compiled.order[: target_index + 1]: + node = graph.node(node_id) + receipt = manifest.nodes[node_id] + if receipt.status == "unreached": + continue + if receipt.status != "executed": + raise StoreCorruptError( + f"Manifest node {node_id!r} has unknown status {receipt.status!r}." + ) + + if node.structural not in { + StructuralDelta.NONE, + StructuralDelta.REVISION, + }: + if receipt.frame_key is None: + raise StoreCorruptError( + f"Structural node {node_id!r} has no stored frame key." + ) + frame = store.load_frame(receipt.frame_key, node_key=receipt.key) + base = None + if node.structural is not StructuralDelta.CREATE: + if node.structural is StructuralDelta.UNION: + base = None + else: + assert node.base is not None + base = populations.get(node.base) + populations[node_id] = Population.from_frame( + frame, + node_id, + _owners_for_stored_frame(frame, node_id=node_id, base=base), + ) + else: + version = compiled.versions[node_id] + if node.structural is StructuralDelta.REVISION: + assert node.base is not None + incumbent = populations[node.base] + else: + incumbent = populations[version] + columns = { + coordinate: store.load_column(key, node_key=receipt.key) + for coordinate, key in receipt.artifacts.items() + } + updated = patch( + incumbent, + node, + KernelResult(columns=columns), + mass_partition=graph.mass_partition, + ) + populations[version] = updated + + target_version = compiled.versions[product.node] + try: + return populations[target_version].frame + except KeyError as error: + raise GraphRuntimeError( + f"Population product {product_name!r} could not be reconstructed." + ) from error diff --git a/packages/microcosm-graph/src/microcosm/graph/runner.py b/packages/microcosm-graph/src/microcosm/graph/runner.py new file mode 100644 index 000000000..9a242d657 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/runner.py @@ -0,0 +1,107 @@ +"""Shared entry point for one YAML root, one compiled graph, and one run.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +from .decl import CompiledGraph, compile_graph +from .executor import run_graph +from .graph_source import LoadedGraphSource, load_graph_source +from .kernel import KernelRegistry +from .manifest import Decision, RunManifest +from .materialize import CandidateIndex, MaterializerRegistry, materialize_products +from .serialize import graph_document_to_json +from .store import ContentStore, ResumePolicy + +__all__ = ["GraphRunResult", "run_graph_source"] + + +@dataclass(frozen=True) +class GraphRunResult: + """Compiled declaration, saved run evidence, and optional local candidate.""" + + source: LoadedGraphSource + compiled: CompiledGraph + manifest: RunManifest + manifest_path: Path + graph_json_path: Path | None + candidate_index: CandidateIndex | None + + +def run_graph_source( + root: str | Path, + *, + sources: Mapping[str, Path], + store: ContentStore, + kernels: KernelRegistry, + manifest_path: str | Path, + parameters: Mapping[str, object] | None = None, + resume: ResumePolicy = "auto", + decisions: tuple[Decision, ...] = (), + graph_json_path: str | Path | None = None, + materializers: MaterializerRegistry | None = None, + candidate_directory: str | Path | None = None, + materialized_outputs: Mapping[str, str] | None = None, +) -> GraphRunResult: + """Compile and execute one selected YAML root, then save local products. + + Post-run materialization starts only after the canonical manifest has been + saved. This helper has no publication or remote-write behavior. + """ + + loaded = load_graph_source(root, parameters=parameters) + compiled = compile_graph(loaded.graph) + graph_json = ( + graph_document_to_json(loaded.graph) if graph_json_path is not None else None + ) + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + decisions=decisions, + graph_source=loaded, + graph_json=graph_json, + ) + saved_manifest = Path(manifest_path) + manifest.save(saved_manifest) + + saved_graph_json: Path | None = None + if graph_json_path is not None: + assert graph_json is not None + saved_graph_json = Path(graph_json_path) + saved_graph_json.parent.mkdir(parents=True, exist_ok=True) + saved_graph_json.write_text(graph_json, encoding="utf-8") + + outputs = {} if materialized_outputs is None else dict(materialized_outputs) + candidate_index: CandidateIndex | None = None + if outputs: + if materializers is None or candidate_directory is None: + raise ValueError( + "materializers and candidate_directory are required when outputs " + "are requested." + ) + candidate_index = materialize_products( + loaded.graph, + saved_manifest, + store, + candidate_directory, + materializers, + outputs, + ) + elif materializers is not None or candidate_directory is not None: + raise ValueError( + "materialized_outputs are required with post-run materialization options." + ) + + return GraphRunResult( + source=loaded, + compiled=compiled, + manifest=manifest, + manifest_path=saved_manifest, + graph_json_path=saved_graph_json, + candidate_index=candidate_index, + ) diff --git a/packages/microcosm-graph/src/microcosm/graph/schema/graph-module-v1.schema.json b/packages/microcosm-graph/src/microcosm/graph/schema/graph-module-v1.schema.json new file mode 100644 index 000000000..3281fba81 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/schema/graph-module-v1.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "graph-module-v1.schema.json", + "type": "object", + "additionalProperties": false, + "required": ["schema_version"], + "properties": { + "schema_version": {"const": 1}, + "modules": {"$ref": "graph-source-v1.schema.json#/$defs/modules"}, + "parameters": {"$ref": "graph-source-v1.schema.json#/$defs/parameters"}, + "sources": {"$ref": "graph-source-v1.schema.json#/$defs/sources"}, + "nodes": {"$ref": "graph-source-v1.schema.json#/$defs/nodes"}, + "products": {"$ref": "graph-source-v1.schema.json#/$defs/products"} + } +} diff --git a/packages/microcosm-graph/src/microcosm/graph/schema/graph-source-v1.schema.json b/packages/microcosm-graph/src/microcosm/graph/schema/graph-source-v1.schema.json new file mode 100644 index 000000000..ee9f68c13 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/schema/graph-source-v1.schema.json @@ -0,0 +1,211 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "graph-source-v1.schema.json", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "country"], + "properties": { + "schema_version": {"const": 1}, + "country": {"$ref": "#/$defs/name"}, + "modules": {"$ref": "#/$defs/modules"}, + "parameters": {"$ref": "#/$defs/parameters"}, + "sources": {"$ref": "#/$defs/sources"}, + "nodes": {"$ref": "#/$defs/nodes"}, + "products": {"$ref": "#/$defs/products"}, + "mass_partition": { + "type": "array", + "prefixItems": [{"$ref": "#/$defs/name"}, {"$ref": "#/$defs/name"}], + "minItems": 2, + "maxItems": 2 + } + }, + "$defs": { + "name": {"type": "string", "minLength": 1}, + "modules": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "uniqueItems": true, + "default": [] + }, + "parameters": { + "type": "object", + "additionalProperties": {"$ref": "#/$defs/parameter"}, + "default": {} + }, + "parameter": { + "type": "object", + "additionalProperties": false, + "required": ["type", "required"], + "properties": { + "type": {"enum": ["boolean", "integer", "number", "string"]}, + "required": {"type": "boolean"}, + "default": {"type": ["boolean", "integer", "number", "string", "null"]}, + "allowed": { + "type": "array", + "items": {"type": ["boolean", "integer", "number", "string", "null"]}, + "minItems": 1, + "uniqueItems": true + }, + "minimum": {"type": "number"}, + "maximum": {"type": "number"}, + "description": {"type": "string"} + } + }, + "sources": { + "type": "array", + "items": {"$ref": "#/$defs/source"}, + "default": [] + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": ["name", "codec"], + "properties": { + "name": {"$ref": "#/$defs/name"}, + "codec": {"$ref": "#/$defs/name"}, + "content_type": {"type": "string", "minLength": 1}, + "access": {"type": "string", "minLength": 1}, + "expected": { + "type": "array", + "items": {"$ref": "#/$defs/expected_content"} + }, + "description": {"type": "string"} + } + }, + "expected_content": { + "type": "object", + "additionalProperties": false, + "required": ["sha256"], + "properties": { + "sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"}, + "boundary": {"enum": ["source", "member"]}, + "path": {"type": "string", "minLength": 1}, + "size": {"type": "integer", "minimum": 0}, + "identity_ref": {"type": "string"} + } + }, + "nodes": { + "type": "array", + "items": {"$ref": "#/$defs/node"}, + "default": [] + }, + "node": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kernel"], + "properties": { + "id": {"$ref": "#/$defs/name"}, + "kernel": {"$ref": "#/$defs/name"}, + "inputs": {"type": "array", "items": {"$ref": "#/$defs/slice"}}, + "outputs": {"type": "array", "items": {"$ref": "#/$defs/owned"}}, + "params": {"type": "object"}, + "param_bindings": { + "type": "object", + "additionalProperties": {"$ref": "#/$defs/name"} + }, + "population": {"$ref": "#/$defs/name"}, + "structural": {"enum": ["none", "create", "filter", "expand", "reweight", "revision", "union"]}, + "base": {"$ref": "#/$defs/name"}, + "bases": {"type": "array", "items": {"$ref": "#/$defs/name"}, "minItems": 2, "uniqueItems": true}, + "sources": {"type": "array", "items": {"$ref": "#/$defs/name"}, "uniqueItems": true}, + "weights": {"$ref": "#/$defs/weights"}, + "mass": {"enum": ["conserve", "free", "declared"]}, + "entrants": {"type": "boolean"}, + "description": {"type": "string"}, + "citation": {"type": "string"}, + "artifact_inputs": {"type": "array", "items": {"$ref": "#/$defs/artifact_input"}}, + "artifact_outputs": {"type": "array", "items": {"$ref": "#/$defs/artifact_output"}}, + "requires_success": {"type": "array", "items": {"$ref": "#/$defs/name"}, "uniqueItems": true} + } + }, + "slice": { + "type": "object", + "additionalProperties": false, + "required": ["entity", "columns"], + "properties": { + "entity": {"$ref": "#/$defs/name"}, + "columns": {"type": "array", "items": {"$ref": "#/$defs/name"}, "minItems": 1, "uniqueItems": true}, + "rows": {"$ref": "#/$defs/name"} + } + }, + "owned": { + "type": "object", + "additionalProperties": false, + "required": ["entity", "column", "dtype"], + "properties": { + "entity": {"$ref": "#/$defs/name"}, + "column": {"$ref": "#/$defs/name"}, + "dtype": {"enum": ["bool", "boolean", "int32", "int64", "Int64", "float32", "float64", "string"]}, + "rows": {"$ref": "#/$defs/name"}, + "ownership": {"enum": ["produced", "absent"]}, + "rewrite": {"type": "boolean"} + } + }, + "weights": { + "type": "object", + "additionalProperties": false, + "required": ["entity", "to_kind"], + "properties": { + "entity": {"$ref": "#/$defs/name"}, + "to_kind": {"enum": ["design", "importance", "calibrated"]}, + "mass": {"enum": ["conserve", "free", "declared"]}, + "anchor": {"$ref": "#/$defs/name"} + } + }, + "artifact_type": { + "type": "object", + "additionalProperties": false, + "required": ["name", "schema_version"], + "properties": { + "name": {"$ref": "#/$defs/name"}, + "schema_version": {"type": "integer", "minimum": 1} + } + }, + "artifact_input": { + "type": "object", + "additionalProperties": false, + "required": ["name", "producer", "artifact", "type"], + "properties": { + "name": {"$ref": "#/$defs/name"}, + "producer": {"$ref": "#/$defs/name"}, + "artifact": {"$ref": "#/$defs/name"}, + "type": {"$ref": "#/$defs/artifact_type"} + } + }, + "artifact_output": { + "type": "object", + "additionalProperties": false, + "required": ["name", "type"], + "properties": { + "name": {"$ref": "#/$defs/name"}, + "type": {"$ref": "#/$defs/artifact_type"} + } + }, + "products": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "kind", "target"], + "properties": { + "name": {"$ref": "#/$defs/name"}, + "kind": {"enum": ["population", "coordinate", "weights", "artifact", "validation", "export"]}, + "target": { + "type": "object", + "additionalProperties": false, + "properties": { + "node": {"$ref": "#/$defs/name"}, + "entity": {"$ref": "#/$defs/name"}, + "column": {"$ref": "#/$defs/name"}, + "artifact": {"$ref": "#/$defs/name"}, + "product": {"$ref": "#/$defs/name"} + } + }, + "codec": {"type": "string"}, + "codec_version": {"type": "integer", "minimum": 1} + } + }, + "default": [] + } + } +} diff --git a/packages/microcosm-graph/src/microcosm/graph/serialize.py b/packages/microcosm-graph/src/microcosm/graph/serialize.py index 85c71b4de..a8a1bac51 100644 --- a/packages/microcosm-graph/src/microcosm/graph/serialize.py +++ b/packages/microcosm-graph/src/microcosm/graph/serialize.py @@ -4,24 +4,36 @@ import json from collections.abc import Mapping +from types import MappingProxyType from .canonical import canonical_json from .decl import ( ArtifactInput, ArtifactOutput, ArtifactType, + ExpectedContent, Graph, Node, Owned, Ownership, Param, + Product, + ProductKind, Slice, SourceRef, StructuralDelta, WeightTransition, ) -__all__ = ["graph_from_json", "graph_to_json"] +__all__ = [ + "GRAPH_DOCUMENT_SERIALIZATION_VERSION", + "graph_document_from_json", + "graph_document_to_json", + "graph_from_json", + "graph_to_json", +] + +GRAPH_DOCUMENT_SERIALIZATION_VERSION = 1 def graph_to_json(graph: Graph) -> str: @@ -36,6 +48,40 @@ def graph_to_json(graph: Graph) -> str: "name": source.name, "codec": source.codec, "description": source.description, + **( + {"content_type": source.content_type} + if source.content_type != "application/octet-stream" + else {} + ), + **({"access": source.access} if source.access is not None else {}), + **( + { + "expected": [ + { + "sha256": item.sha256, + **( + {"boundary": item.boundary} + if item.boundary != "source" + else {} + ), + **( + {"path": item.path} if item.path is not None else {} + ), + **( + {"size": item.size} if item.size is not None else {} + ), + **( + {"identity_ref": item.identity_ref} + if item.identity_ref + else {} + ), + } + for item in source.expected + ] + } + if source.expected + else {} + ), } for source in graph.sources ], @@ -47,6 +93,11 @@ def graph_to_json(graph: Graph) -> str: if graph.mass_partition is None else {"mass_partition": list(graph.mass_partition)} ), + **( + {} + if not graph.products + else {"products": [_product_payload(product) for product in graph.products]} + ), } return canonical_json(payload).decode("utf-8") @@ -64,6 +115,8 @@ def graph_from_json(text: str) -> Graph: fields = {"country", "sources", "nodes"} if "mass_partition" in root: fields.add("mass_partition") + if "products" in root: + fields.add("products") _exact_fields(root, fields, "graph") sources_raw = _array(root["sources"], "graph.sources") nodes_raw = _array(root["nodes"], "graph.nodes") @@ -79,6 +132,63 @@ def graph_from_json(text: str) -> Graph: mass_partition=_partition_from_payload( root.get("mass_partition"), "graph.mass_partition" ), + products=tuple( + _product_from_payload(value, index) + for index, value in enumerate( + _array(root.get("products", []), "graph.products") + ) + ), + ) + + +def graph_document_to_json(graph: Graph) -> str: + """Return a versioned generated document without changing legacy bytes.""" + + payload = { + "derived": True, + "document_type": "microcosm.graph", + "graph": json.loads(graph_to_json(graph)), + "serialization_version": GRAPH_DOCUMENT_SERIALIZATION_VERSION, + } + return json.dumps(payload, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def graph_document_from_json(text: str) -> Graph: + """Read the closed envelope emitted by :func:`graph_document_to_json`.""" + + def unique(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON field {key!r}") + result[key] = value + return result + + try: + payload = json.loads( + text, object_pairs_hook=unique, parse_constant=_reject_json_constant + ) + except json.JSONDecodeError as error: + raise ValueError(f"Invalid generated Graph JSON: {error.msg}.") from error + mapping = _mapping(payload, "generated graph document") + _exact_fields( + mapping, + {"derived", "document_type", "graph", "serialization_version"}, + "generated graph document", + ) + if mapping["serialization_version"] != GRAPH_DOCUMENT_SERIALIZATION_VERSION: + raise ValueError( + "Unsupported generated Graph JSON serialization version " + f"{mapping['serialization_version']!r}." + ) + if mapping["derived"] is not True or mapping["document_type"] != "microcosm.graph": + raise ValueError( + "Generated Graph JSON is not a derived microcosm.graph document." + ) + return graph_from_json( + json.dumps( + mapping["graph"], sort_keys=True, separators=(",", ":"), allow_nan=False + ) ) @@ -152,6 +262,7 @@ def _node_payload(node: Node) -> dict[str, object]: "population": node.population, "structural": node.structural.value, "base": node.base, + **({"bases": list(node.bases)} if node.bases else {}), "sources": list(node.sources), "weights": ( None @@ -160,10 +271,20 @@ def _node_payload(node: Node) -> dict[str, object]: "entity": node.weights.entity, "to_kind": node.weights.to_kind, "mass": node.weights.mass, + **( + {"anchor": node.weights.anchor} + if node.weights.anchor is not None + else {} + ), } ), "mass": node.mass, **({"entrants": True} if node.entrants else {}), + **( + {"requires_success": list(node.requires_success)} + if node.requires_success + else {} + ), "description": node.description, "citation": node.citation, } @@ -172,11 +293,44 @@ def _node_payload(node: Node) -> dict[str, object]: def _source_from_payload(value: object, index: int) -> SourceRef: label = f"graph.sources[{index}]" payload = _mapping(value, label) - _exact_fields(payload, {"name", "codec", "description"}, label) + fields = {"name", "codec", "description"} + fields.update( + name for name in ("content_type", "access", "expected") if name in payload + ) + _exact_fields(payload, fields, label) + expected = _array(payload.get("expected", []), f"{label}.expected") return SourceRef( name=_string(payload["name"], f"{label}.name"), codec=_string(payload["codec"], f"{label}.codec"), description=_string(payload["description"], f"{label}.description"), + content_type=_string( + payload.get("content_type", "application/octet-stream"), + f"{label}.content_type", + ), + access=_optional_string(payload.get("access"), f"{label}.access"), + expected=tuple( + _expected_content_from_payload(item, f"{label}.expected[{item_index}]") + for item_index, item in enumerate(expected) + ), + ) + + +def _expected_content_from_payload(value: object, label: str) -> ExpectedContent: + payload = _mapping(value, label) + fields = {"sha256"} + fields.update( + name for name in ("boundary", "path", "size", "identity_ref") if name in payload + ) + _exact_fields(payload, fields, label) + size = payload.get("size") + if size is not None and (type(size) is not int or size < 0): + raise TypeError(f"{label}.size must be a non-negative integer") + return ExpectedContent( + sha256=_string(payload["sha256"], f"{label}.sha256"), + boundary=_string(payload.get("boundary", "source"), f"{label}.boundary"), + path=_optional_string(payload.get("path"), f"{label}.path"), + size=size, + identity_ref=_string(payload.get("identity_ref", ""), f"{label}.identity_ref"), ) @@ -203,6 +357,10 @@ def _node_from_payload(value: object, index: int) -> Node: ) if "entrants" in payload: fields.add("entrants") + if "bases" in payload: + fields.add("bases") + if "requires_success" in payload: + fields.add("requires_success") _exact_fields(payload, fields, label) entrants = payload.get("entrants", False) if not isinstance(entrants, bool): @@ -213,6 +371,10 @@ def _node_from_payload(value: object, index: int) -> Node: params = _mapping(payload["params"], f"{label}.params") population = _optional_string(payload["population"], f"{label}.population") base = _optional_string(payload["base"], f"{label}.base") + bases = _array(payload.get("bases", []), f"{label}.bases") + requires_success = _array( + payload.get("requires_success", []), f"{label}.requires_success" + ) return Node( artifact_inputs=tuple( _artifact_from_payload(value, input_=True) @@ -247,6 +409,10 @@ def _node_from_payload(value: object, index: int) -> Node: _string(payload["structural"], f"{label}.structural") ), base=base, + bases=tuple( + _string(name, f"{label}.bases[{base_index}]") + for base_index, name in enumerate(bases) + ), sources=tuple( _string(name, f"{label}.sources[{source_index}]") for source_index, name in enumerate(sources) @@ -256,6 +422,10 @@ def _node_from_payload(value: object, index: int) -> Node: entrants=entrants, description=_string(payload["description"], f"{label}.description"), citation=_string(payload["citation"], f"{label}.citation"), + requires_success=tuple( + _string(name, f"{label}.requires_success[{item_index}]") + for item_index, name in enumerate(requires_success) + ), ) @@ -296,11 +466,15 @@ def _weights_from_payload(value: object, label: str) -> WeightTransition | None: if value is None: return None payload = _mapping(value, label) - _exact_fields(payload, {"entity", "to_kind", "mass"}, label) + fields = {"entity", "to_kind", "mass"} + if "anchor" in payload: + fields.add("anchor") + _exact_fields(payload, fields, label) return WeightTransition( entity=_string(payload["entity"], f"{label}.entity"), to_kind=_string(payload["to_kind"], f"{label}.to_kind"), mass=_string(payload["mass"], f"{label}.mass"), + anchor=_optional_string(payload.get("anchor"), f"{label}.anchor"), ) @@ -314,9 +488,68 @@ def _param_from_json(value: object, label: str) -> Param: _param_from_json(child, f"{label}[{index}]") for index, child in enumerate(value) ) + if isinstance(value, Mapping): + return MappingProxyType( + { + _string(key, f"{label} key"): _param_from_json( + child, f"{label}[{key!r}]" + ) + for key, child in value.items() + } + ) raise TypeError(f"{label} is not a legal graph parameter") +def _product_payload(product: Product) -> dict[str, object]: + return { + "name": product.name, + "kind": product.kind.value, + **({"node": product.node} if product.node is not None else {}), + **({"entity": product.entity} if product.entity is not None else {}), + **({"column": product.column} if product.column is not None else {}), + **({"artifact": product.artifact} if product.artifact is not None else {}), + **({"source": product.source} if product.source is not None else {}), + **({"codec": product.codec} if product.codec is not None else {}), + **( + {"codec_version": product.codec_version} + if product.codec_version is not None + else {} + ), + } + + +def _product_from_payload(value: object, index: int) -> Product: + label = f"graph.products[{index}]" + payload = _mapping(value, label) + fields = {"name", "kind"} | ( + set(payload) + & { + "node", + "entity", + "column", + "artifact", + "source", + "codec", + "codec_version", + } + ) + _exact_fields(payload, fields, label) + version = payload.get("codec_version") + if version is not None and (type(version) is not int or version < 1): + raise TypeError(f"{label}.codec_version must be a positive integer") + return Product( + name=_string(payload["name"], f"{label}.name"), + kind=ProductKind(_string(payload["kind"], f"{label}.kind")), + node=_optional_string(payload.get("node"), f"{label}.node"), + entity=_optional_string(payload.get("entity"), f"{label}.entity"), + column=_optional_string(payload.get("column"), f"{label}.column"), + artifact=_optional_string(payload.get("artifact"), f"{label}.artifact"), + source=_optional_string(payload.get("source"), f"{label}.source"), + codec=_optional_string(payload.get("codec"), f"{label}.codec"), + codec_version=version, + ) + + def _mapping(value: object, label: str) -> Mapping[str, object]: if not isinstance(value, Mapping) or not all( isinstance(name, str) for name in value diff --git a/packages/microcosm-graph/src/microcosm/graph/source_errors.py b/packages/microcosm-graph/src/microcosm/graph/source_errors.py new file mode 100644 index 000000000..009905f17 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/source_errors.py @@ -0,0 +1,60 @@ +"""Stable errors raised while loading an authored graph source.""" + +from __future__ import annotations + + +class GraphSourceError(ValueError): + """Base error with an editor-ready source location and JSON Pointer.""" + + def __init__( + self, + message: str, + *, + source: str | None = None, + pointer: str | None = None, + line: int | None = None, + column: int | None = None, + ) -> None: + self.message = message + self.source = source + self.pointer = pointer + self.line = line + self.column = column + location = source + if location is not None and line is not None: + location += f":{line}" + if column is not None: + location += f":{column}" + if pointer: + location = f"{location or ''} {pointer}".strip() + super().__init__(f"{location}: {message}" if location else message) + + +class GraphSourceParseError(GraphSourceError): + """Authored input is outside the accepted YAML/JSON subset.""" + + +class GraphSourceSchemaError(GraphSourceError): + """The packaged graph-source schema catalog is invalid.""" + + +class GraphSourceValidationError(GraphSourceError): + """A parsed graph source does not satisfy its closed schema.""" + + +class GraphSourceCompositionError(GraphSourceError): + """Declared graph modules cannot be composed safely.""" + + +class GraphParameterBindingError(GraphSourceError): + """Run parameter declarations and supplied bindings disagree.""" + + +__all__ = [ + "GraphParameterBindingError", + "GraphSourceCompositionError", + "GraphSourceError", + "GraphSourceParseError", + "GraphSourceSchemaError", + "GraphSourceValidationError", +] diff --git a/packages/microcosm-graph/src/microcosm/graph/store.py b/packages/microcosm-graph/src/microcosm/graph/store.py index 6f93b8b65..b42895138 100644 --- a/packages/microcosm-graph/src/microcosm/graph/store.py +++ b/packages/microcosm-graph/src/microcosm/graph/store.py @@ -46,6 +46,8 @@ StoreMissError, StoreUnavailableError, ) +from .keys import artifact_key +from .keys import frame_key as node_frame_key __all__ = [ "ContentStore", @@ -70,6 +72,7 @@ _ENCODING_NULLABLE_INTEGER = "nullable-integer-v1" _ENCODING_UTF8 = "utf8-offsets-v1" _ENCODING_OBJECT = "object-scalars-v1" +_ENCODING_FRAME_COLUMN_REF = "frame-column-ref-v1" _TAG_NONE = 0 _TAG_PD_NA = 1 @@ -829,6 +832,47 @@ def build(root: Path) -> Mapping[str, object]: write_column = put_column + def put_frame_column_ref( + self, + key: str, + *, + frame_key: str, + entity: str, + column: str, + series: pd.Series, + declared_dtype: str, + node_key: str, + verify_existing: bool = True, + ) -> Path: + """Store a small coordinate reference instead of duplicating frame values.""" + + if verify_existing: + frame_metadata = self.metadata(frame_key, kind="frame") + if frame_metadata.get("node_key") != node_key: + raise ValueError(f"Stored frame {frame_key} belongs to another node.") + if not isinstance(series, pd.Series): + raise TypeError("Frame column references require a pandas Series.") + if not _dtype_matches_declared(series.dtype, declared_dtype): + raise TypeError( + f"Frame coordinate {entity}.{column} has dtype {series.dtype!s}, " + f"not {declared_dtype!r}." + ) + + def build(root: Path) -> Mapping[str, object]: + del root + return { + "encoding": _ENCODING_FRAME_COLUMN_REF, + "frame_key": frame_key, + "entity": entity, + "column": column, + "declared_dtype": declared_dtype, + "pandas_dtype": str(series.dtype), + "length": len(series), + "node_key": node_key, + } + + return self._put(key, "column", build, verify_existing=verify_existing) + def load_column( self, key: str, @@ -850,6 +894,55 @@ def load_column( ) if node_key is not None and metadata.get("node_key") != node_key: raise StoreCorrupt(f"Stored column {key} belongs to a different node.") + if metadata.get("encoding") == _ENCODING_FRAME_COLUMN_REF: + frame_key = metadata.get("frame_key") + entity = metadata.get("entity") + column = metadata.get("column") + if not all( + isinstance(value, str) and value + for value in (frame_key, entity, column) + ): + raise StoreCorrupt(f"Stored column reference {key} is malformed.") + if frame_key == key: + raise StoreCorrupt(f"Stored column reference {key} refers to itself.") + reference_node_key = metadata.get("node_key") + if ( + not isinstance(reference_node_key, str) + or artifact_key(reference_node_key, entity, column) != key + or node_frame_key(reference_node_key) != frame_key + ): + raise StoreCorrupt( + f"Stored column reference {key} disagrees with its node identity." + ) + frame = self.load_frame(frame_key, node_key=reference_node_key) + if entity not in frame.entities or column not in frame.table(entity): + raise StoreCorrupt( + f"Stored column reference {key} names absent {entity}.{column}." + ) + table = frame.table(entity) + id_column = frame.schema.entity_id_column(entity) + values = pd.Series( + table[column].array.copy(), + index=pd.Index(table[id_column].array.copy(), name=id_column), + name=column, + dtype=table[column].dtype, + ) + if ( + str(values.dtype) != metadata.get("pandas_dtype") + or metadata.get("length") != len(values) + or not _dtype_matches_declared(values.dtype, stored_dtype) + ): + raise StoreCorrupt( + f"Stored column reference {key} disagrees with its frame." + ) + if entity_ids is not None: + if isinstance(entity_ids, pd.Series): + expected_ids = pd.Index(entity_ids.array, name=entity_ids.name) + else: + expected_ids = pd.Index(entity_ids) + if not values.index.identical(expected_ids): + raise StoreCorrupt(f"Stored column {key} has different entity ids.") + return values value_spec = metadata.get("values") ids_spec = metadata.get("ids") if not isinstance(value_spec, dict) or not isinstance(ids_spec, dict): diff --git a/packages/microcosm-graph/src/microcosm/graph/yaml12.py b/packages/microcosm-graph/src/microcosm/graph/yaml12.py new file mode 100644 index 000000000..d94336f56 --- /dev/null +++ b/packages/microcosm-graph/src/microcosm/graph/yaml12.py @@ -0,0 +1,370 @@ +"""Parse the deterministic JSON-compatible YAML 1.2 subset.""" + +from __future__ import annotations + +import json +import math +import re +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from os import PathLike +from pathlib import Path +from types import MappingProxyType + +import yaml +from yaml.composer import ComposerError +from yaml.error import Mark, YAMLError +from yaml.loader import SafeLoader +from yaml.nodes import MappingNode, Node, ScalarNode, SequenceNode +from yaml.tokens import DirectiveToken, TagToken + +from .source_errors import GraphSourceParseError + +type JSONScalar = None | bool | int | float | str +type JSONValue = JSONScalar | list[JSONValue] | dict[str, JSONValue] + +_BOOL_TAG = "tag:yaml.org,2002:bool" +_FLOAT_TAG = "tag:yaml.org,2002:float" +_INT_TAG = "tag:yaml.org,2002:int" +_MERGE_TAG = "tag:yaml.org,2002:merge" +_NULL_TAG = "tag:yaml.org,2002:null" +_STR_TAG = "tag:yaml.org,2002:str" +_TIMESTAMP_TAG = "tag:yaml.org,2002:timestamp" +_JSON_SCALAR_TAGS = frozenset({_BOOL_TAG, _FLOAT_TAG, _INT_TAG, _NULL_TAG, _STR_TAG}) + + +@dataclass(frozen=True) +class SourceLocation: + source: str + line: int + column: int + + +@dataclass(frozen=True) +class ParsedYAML: + value: JSONValue + locations: Mapping[str, SourceLocation] + + +class _StrictYAML12Loader(SafeLoader): + """SafeLoader with YAML 1.2 core boolean and numeric resolution.""" + + +_StrictYAML12Loader.yaml_implicit_resolvers = { + first: list(resolvers) + for first, resolvers in SafeLoader.yaml_implicit_resolvers.items() +} +for _first, _resolvers in tuple(_StrictYAML12Loader.yaml_implicit_resolvers.items()): + _StrictYAML12Loader.yaml_implicit_resolvers[_first] = [ + (tag, regexp) + for tag, regexp in _resolvers + if tag not in {_BOOL_TAG, _FLOAT_TAG, _INT_TAG} + ] +_StrictYAML12Loader.add_implicit_resolver( + _BOOL_TAG, + re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$"), + list("tTfF"), +) +_StrictYAML12Loader.add_implicit_resolver( + _INT_TAG, + re.compile( + r"^(?:[-+]?0b[0-1_]+|[-+]?0o[0-7_]+|" + r"[-+]?0x[0-9a-fA-F_]+|[-+]?[0-9][0-9_]*)$" + ), + list("-+0123456789"), +) +_StrictYAML12Loader.add_implicit_resolver( + _FLOAT_TAG, + re.compile( + r"^(?:[-+]?(?:[0-9][0-9_]*\.[0-9_]*|\.[0-9_]+)" + r"(?:[eE][-+]?[0-9]+)?|[-+]?[0-9][0-9_]*(?:[eE][-+]?[0-9]+)|" + r"[-+]?\.(?:inf|Inf|INF)|[-+]?\.(?:nan|NaN|NAN))$" + ), + list("-+0123456789."), +) + + +def _construct_yaml12_int(loader: SafeLoader, node: ScalarNode) -> int: + value = loader.construct_scalar(node).replace("_", "") + sign = -1 if value.startswith("-") else 1 + unsigned = value[1:] if value[:1] in {"+", "-"} else value + for prefix, base in (("0b", 2), ("0o", 8), ("0x", 16)): + if unsigned.startswith(prefix): + return sign * int(unsigned[2:], base) + return sign * int(unsigned, 10) + + +_StrictYAML12Loader.add_constructor(_INT_TAG, _construct_yaml12_int) + + +def _error( + message: str, *, source: str, mark: Mark | None = None, pointer: str | None = None +) -> GraphSourceParseError: + return GraphSourceParseError( + message, + source=source, + pointer=pointer, + line=None if mark is None else mark.line + 1, + column=None if mark is None else mark.column + 1, + ) + + +def _marked_yaml_error(exc: YAMLError, *, source: str) -> GraphSourceParseError: + mark = getattr(exc, "problem_mark", None) or getattr(exc, "context_mark", None) + if isinstance(exc, ComposerError) and "single document" in str(exc): + message = "multiple YAML documents are not allowed" + else: + problem = getattr(exc, "problem", None) + message = f"invalid YAML: {problem}" if problem else "invalid YAML" + return _error(message, source=source, mark=mark) + + +def _tokens(text: str, *, source: str) -> Iterator[object]: + try: + yield from yaml.scan(text, Loader=_StrictYAML12Loader) + except YAMLError as exc: + raise _marked_yaml_error(exc, source=source) from exc + + +def _reject_syntax_extensions(text: str, *, source: str) -> None: + for token in _tokens(text, source=source): + if isinstance(token, TagToken): + raise _error( + "explicit YAML tags are not allowed", + source=source, + mark=token.start_mark, + ) + if isinstance(token, DirectiveToken): + if token.name == "TAG": + raise _error( + "YAML tag directives are not allowed", + source=source, + mark=token.start_mark, + ) + if token.name == "YAML" and token.value != (1, 2): + raise _error( + "only the YAML 1.2 directive is allowed", + source=source, + mark=token.start_mark, + ) + if token.name not in {"TAG", "YAML"}: + raise _error( + "YAML directives other than %YAML 1.2 are not allowed", + source=source, + mark=token.start_mark, + ) + + +def _pointer(parent: str, token: object) -> str: + escaped = str(token).replace("~", "~0").replace("/", "~1") + return f"{parent}/{escaped}" if parent else f"/{escaped}" + + +def _validate_node( + node: Node, + *, + source: str, + active: set[int], + validated: set[int], +) -> None: + identity = id(node) + if identity in active: + raise _error( + "cyclic YAML aliases are not allowed", source=source, mark=node.start_mark + ) + if identity in validated: + return + active.add(identity) + try: + if isinstance(node, ScalarNode): + if node.tag == _TIMESTAMP_TAG: + raise _error( + "timestamps and dates are not allowed", + source=source, + mark=node.start_mark, + ) + if node.tag not in _JSON_SCALAR_TAGS: + raise _error( + "only JSON-compatible scalar values are allowed", + source=source, + mark=node.start_mark, + ) + if node.tag == _FLOAT_TAG: + normalized = node.value.replace("_", "").lower() + try: + finite = normalized.lstrip("+-") not in { + ".inf", + ".nan", + } and math.isfinite(float(normalized)) + except ValueError: + raise _error( + "invalid YAML 1.2 number", source=source, mark=node.start_mark + ) from None + if not finite: + raise _error( + "non-finite numbers are not allowed", + source=source, + mark=node.start_mark, + ) + if node.tag == _INT_TAG: + normalized = node.value.replace("_", "").lstrip("+-") + if normalized.startswith(("0b", "0o", "0x")): + normalized = normalized[2:] + if not normalized: + raise _error( + "invalid YAML 1.2 number", + source=source, + mark=node.start_mark, + ) + return + if isinstance(node, SequenceNode): + for item in node.value: + _validate_node(item, source=source, active=active, validated=validated) + return + if isinstance(node, MappingNode): + keys: set[str] = set() + for key_node, value_node in node.value: + if key_node.tag == _MERGE_TAG: + raise _error( + "YAML merge keys are not allowed", + source=source, + mark=key_node.start_mark, + ) + if not isinstance(key_node, ScalarNode) or key_node.tag != _STR_TAG: + raise _error( + "mapping keys must be strings", + source=source, + mark=key_node.start_mark, + ) + if key_node.value in keys: + raise _error( + f"duplicate mapping key {key_node.value!r}", + source=source, + mark=key_node.start_mark, + ) + keys.add(key_node.value) + _validate_node( + value_node, source=source, active=active, validated=validated + ) + return + raise _error( + "only JSON-compatible YAML nodes are allowed", + source=source, + mark=node.start_mark, + ) + finally: + active.remove(identity) + validated.add(identity) + + +def _source_map( + node: Node, *, source: str, pointer: str, result: dict[str, SourceLocation] +) -> None: + result.setdefault( + pointer or "/", + SourceLocation(source, node.start_mark.line + 1, node.start_mark.column + 1), + ) + if isinstance(node, SequenceNode): + for index, child in enumerate(node.value): + _source_map( + child, source=source, pointer=_pointer(pointer, index), result=result + ) + elif isinstance(node, MappingNode): + for key, child in node.value: + child_pointer = _pointer(pointer, key.value) + result[child_pointer] = SourceLocation( + source, key.start_mark.line + 1, key.start_mark.column + 1 + ) + _source_map(child, source=source, pointer=child_pointer, result=result) + + +def _ensure_json_value(value: object, *, source: str) -> JSONValue: + if value is None or isinstance(value, str | bool | int): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise _error("non-finite numbers are not allowed", source=source) + return value + if isinstance(value, list): + return [_ensure_json_value(item, source=source) for item in value] + if isinstance(value, dict): + if not all(isinstance(key, str) for key in value): + raise _error("mapping keys must be strings", source=source) + return { + key: _ensure_json_value(item, source=source) for key, item in value.items() + } + raise _error("only JSON-compatible values are allowed", source=source) + + +def parse_yaml12(text: str, *, source: str = "") -> ParsedYAML: + """Load one YAML document and retain stable locations for its values.""" + if not isinstance(text, str): + raise TypeError("YAML input must be text") + _reject_syntax_extensions(text, source=source) + loader = _StrictYAML12Loader(text) + try: + node = loader.get_single_node() + if node is None: + return ParsedYAML(None, MappingProxyType({})) + _validate_node(node, source=source, active=set(), validated=set()) + locations: dict[str, SourceLocation] = {} + _source_map(node, source=source, pointer="", result=locations) + value = _ensure_json_value(loader.construct_document(node), source=source) + return ParsedYAML(value, MappingProxyType(locations)) + except GraphSourceParseError: + raise + except YAMLError as exc: + raise _marked_yaml_error(exc, source=source) from exc + finally: + loader.dispose() + + +def load_yaml12(text: str, *, source: str = "") -> JSONValue: + return parse_yaml12(text, source=source).value + + +def load_yaml12_file(path: str | PathLike[str]) -> JSONValue: + resource = Path(path) + return load_yaml12(resource.read_text(encoding="utf-8"), source=str(resource)) + + +def load_json_strict(text: str, *, source: str = "") -> JSONValue: + if not isinstance(text, str): + raise TypeError("JSON input must be text") + + def refuse_duplicates(pairs: list[tuple[str, JSONValue]]) -> dict[str, JSONValue]: + result: dict[str, JSONValue] = {} + for key, value in pairs: + if key in result: + raise _error(f"duplicate mapping key {key!r}", source=source) + result[key] = value + return result + + def refuse_constant(_constant: str) -> JSONValue: + raise _error("non-finite numbers are not allowed", source=source) + + try: + return json.loads( + text, object_pairs_hook=refuse_duplicates, parse_constant=refuse_constant + ) + except GraphSourceParseError: + raise + except json.JSONDecodeError as exc: + raise GraphSourceParseError( + f"invalid JSON: {exc.msg}", + source=source, + line=exc.lineno, + column=exc.colno, + ) from exc + + +__all__ = [ + "JSONScalar", + "JSONValue", + "ParsedYAML", + "SourceLocation", + "load_json_strict", + "load_yaml12", + "load_yaml12_file", + "parse_yaml12", +] diff --git a/packages/microcosm-graph/tests/_toy.py b/packages/microcosm-graph/tests/_toy.py index ada684038..972f789c9 100644 --- a/packages/microcosm-graph/tests/_toy.py +++ b/packages/microcosm-graph/tests/_toy.py @@ -278,7 +278,7 @@ class SourceCsv(ToyKernel): """CREATE: the ``csv-tables`` codec turned into a population version.""" def compute(self, context: KernelContext) -> KernelResult: - frame = read_toy_frame(context.sources["survey"]) + frame = context.sources["survey"].decode() return KernelResult( frame=frame, receipt={"persons": frame.n("person"), "households": frame.n("household")}, diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json index 8485a3916..133d54717 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/calibrate/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"a7a0330fb7b7a4c80b62e2a44dcc9075aa6da9b27a20a4282ca6969f4570ef18","kernel":"calibrate.adam@1","node":"calibrate","node_key":"0168f69547e21ef71c67442a52622de83efd32fd23adc8faf93cc347f6c80b4c","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"0168f69547e21ef71c67442a52622de83efd32fd23adc8faf93cc347f6c80b4c"}},"seed":0} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","scipy":"1.17.1","torch":"2.12.0"},"implementation_hash":"a7a0330fb7b7a4c80b62e2a44dcc9075aa6da9b27a20a4282ca6969f4570ef18","kernel":"calibrate.adam@1","node":"calibrate","node_key":"987cf76bade6fbbe28d3301f4b1937641d577eb1beb1ede2e3655f51989d8b74","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"987cf76bade6fbbe28d3301f4b1937641d577eb1beb1ede2e3655f51989d8b74"}},"seed":0} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json index 3b61d6fc1..7fa2c07bb 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/fit.qrf/pins.json @@ -1 +1 @@ -{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","quantile-forest":"1.4.2","scikit-learn":"1.8.0"},"implementation_hash":"d1f8b1929e6452aa507b0d9c64ba42a59851dc31c71021303c25f060e34c075b","kernel":"fit.qrf@1","node":"fit_qrf","node_key":"35af6a452bd32ca39d313d78255236df01877d684c2d847b1bd5a6016a68e237","numeric":"platform_bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"35af6a452bd32ca39d313d78255236df01877d684c2d847b1bd5a6016a68e237"},"x86_64/linux/py3.13":{"direct":"platforms/x86_64-linux-py3_13/direct.csv","node_key":"6c43edcb3edf2bdef20d915b1be16503d37583c678aced78f1442aca1139e1ae"},"x86_64/linux/py3.14":{"direct":"platforms/x86_64-linux-py3_14/direct.csv","node_key":"ecb6b20c02b0bc442c4baf3fd7def6d6aec06e6b9567910294f945d66c43bbf8"}},"seed":947} +{"dependencies":{"numpy":"2.4.6","pandas":"3.0.3","quantile-forest":"1.4.2","scikit-learn":"1.8.0"},"implementation_hash":"d1f8b1929e6452aa507b0d9c64ba42a59851dc31c71021303c25f060e34c075b","kernel":"fit.qrf@1","node":"fit_qrf","node_key":"1c3b437984a6e3ddb85648bff38be754a2d1bade78ba35baaa0bc25e3d768dd4","numeric":"platform_bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"1c3b437984a6e3ddb85648bff38be754a2d1bade78ba35baaa0bc25e3d768dd4"},"x86_64/linux/py3.13":{"direct":"platforms/x86_64-linux-py3_13/direct.csv","node_key":"f5202d8bfd301402a2bf0e9981020cbcef3e302c51fe0437f5644e6f3ada2a62"},"x86_64/linux/py3.14":{"direct":"platforms/x86_64-linux-py3_14/direct.csv","node_key":"94ada5a3c56f6d5f8138103f2f6fb4f1b150e58f0cf635c5a835d4889eaa9032"}},"seed":947} diff --git a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json index 15f5d879b..0bf9e9c49 100644 --- a/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json +++ b/packages/microcosm-graph/tests/fixtures/parity/kernels/simulate/pins.json @@ -1 +1 @@ -{"dependencies":{},"implementation_hash":"eed54eaf27b53faf446068aa833ec16ab398840faf870445dbdcd6a4604c566c","kernel":"simulate.rules@1","node":"simulate","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"a643736e821e841fa74996f0768d61110fe4d6e590d8db4286fd241f7d75b123"}},"seed":null} +{"dependencies":{},"implementation_hash":"eed54eaf27b53faf446068aa833ec16ab398840faf870445dbdcd6a4604c566c","kernel":"simulate.rules@1","node":"simulate","node_key":"d48d2e455778cade49260b3d7fb9ac275b0b9091958de41508f096017d4e3ff7","numeric":"bitwise","platform":"arm64/darwin/py3.14","platforms":{"arm64/darwin/py3.14":{"direct":"direct.csv","node_key":"d48d2e455778cade49260b3d7fb9ac275b0b9091958de41508f096017d4e3ff7"}},"seed":null} diff --git a/packages/microcosm-graph/tests/test_acceptance_b_ownership.py b/packages/microcosm-graph/tests/test_acceptance_b_ownership.py index feefb7b16..eb185c2bc 100644 --- a/packages/microcosm-graph/tests/test_acceptance_b_ownership.py +++ b/packages/microcosm-graph/tests/test_acceptance_b_ownership.py @@ -119,6 +119,7 @@ def test_b2_executor_enforces_ownership(tmp_path: Path) -> None: "tolerances", # amendment 13: declared tolerances of the inputs' owners "numerics", # amendment 17: per-coordinate numeric class, bound, platform "artifacts", # amendment 19: declared immutable typed bytes + "weight_anchors", # declared earlier weight products, keyed by product name } graph = toy.small_graph( diff --git a/packages/microcosm-graph/tests/test_acceptance_h_parity.py b/packages/microcosm-graph/tests/test_acceptance_h_parity.py index 44698f765..01a009f87 100644 --- a/packages/microcosm-graph/tests/test_acceptance_h_parity.py +++ b/packages/microcosm-graph/tests/test_acceptance_h_parity.py @@ -15,6 +15,7 @@ from __future__ import annotations +import hashlib import importlib.util import json import sys @@ -22,6 +23,7 @@ import numpy as np import pytest +import yaml from microcosm.graph import platform_fingerprint @@ -75,6 +77,50 @@ def _assert_same_bytes(actual, expected) -> None: assert np.array_equal(actual.isna().to_numpy(), expected.isna().to_numpy()) +def _assert_yaml_and_generated_json_equivalence(graph, path: Path) -> None: + """Represent one existing Python fixture through both source formats.""" + + from microcosm.graph import ( + graph_document_from_json, + graph_document_to_json, + graph_key, + graph_to_json, + load_graph_source, + ) + + payload = json.loads(graph_to_json(graph)) + payload["schema_version"] = 1 + for node in payload["nodes"]: + for optional in ("population", "base", "weights"): + if node.get(optional) is None: + node.pop(optional) + products = [] + for product in payload.get("products", []): + target = { + ("product" if key == "source" else key): product.pop(key) + for key in ("node", "entity", "column", "artifact", "source") + if key in product + } + products.append({**product, "target": target}) + if products: + payload["products"] = products + path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + + loaded = load_graph_source(path).graph + assert graph_key(loaded) == graph_key(graph) + assert {source.name: source for source in loaded.sources} == { + source.name: source for source in graph.sources + } + assert {node.id: node for node in loaded.nodes} == { + node.id: node for node in graph.nodes + } + assert {product.name: product for product in loaded.products} == { + product.name: product for product in graph.products + } + generated = graph_document_to_json(graph) + assert graph_document_from_json(generated) == graph + + def _frame_differences(actual, expected) -> str: """Name the cells two frames disagree on; two identities alone say nothing.""" import pandas as pd @@ -278,6 +324,7 @@ def test_h2_uk_spine_parity(tmp_path: Path) -> None: # The graph the UK lane ships is also pinned as JSON beside the fixture, so # a silent change to the declaration shows up as a fixture diff. graph = uk_spine_graph() + _assert_yaml_and_generated_json_equivalence(graph, tmp_path / "uk-graph.yaml") assert graph_from_json((UK_SPINE_PARITY / "uk_spine.json").read_text()) == graph compiled = compile_graph(graph) assert len(compiled.order) >= 29, "a CREATE node plus the 28 spine stages" @@ -340,15 +387,17 @@ def test_h3_us_post_transfer_parity(tmp_path: Path) -> None: # The graph the US lane ships is pinned as JSON beside the fixture, so a # silent change to the declaration shows up as a fixture diff. graph = us_post_transfer_graph() + _assert_yaml_and_generated_json_equivalence(graph, tmp_path / "us-graph.yaml") assert ( graph_from_json((US_POST_TRANSFER_PARITY / "us_post_transfer.json").read_text()) == graph ) compiled = compile_graph(graph) + graph_store = ContentStore(tmp_path / "store") manifest = run_graph( compiled, sources={"stacked": US_POST_TRANSFER_PARITY / "sources"}, - store=ContentStore(tmp_path / "store"), + store=graph_store, kernels=us_registry(), resume="forbid", decisions=(), @@ -368,6 +417,49 @@ def test_h3_us_post_transfer_parity(tmp_path: Path) -> None: column ) + # Storage benchmark on the representative synthetic US population. A + # structural frame holds the values once; its coordinate objects are + # metadata-only references. Ordinary value patches retain their exact + # standalone payloads for reconstruction and investigation. + structural_frame_payload = 0 + structural_reference_payload = 0 + duplicated_coordinate_payload = 0 + ordinary_patch_payload = 0 + comparison_store = ContentStore(tmp_path / "duplicated-coordinate-store") + for node_id in compiled.order: + declaration = graph.node(node_id) + receipt = manifest.nodes[node_id] + if receipt.frame_key is not None: + frame_metadata = graph_store.metadata(receipt.frame_key, kind="frame") + structural_frame_payload += sum( + int(entry["size"]) for entry in frame_metadata["payloads"].values() + ) + for key in receipt.artifacts.values(): + metadata = graph_store.metadata(key, kind="column") + payload = sum(int(entry["size"]) for entry in metadata["payloads"].values()) + if declaration.structural.value not in {"none", "revision"}: + assert metadata["encoding"] == "frame-column-ref-v1" + structural_reference_payload += payload + series = graph_store.load_column(key, node_key=receipt.key) + comparison_key = hashlib.sha256(f"{node_id}:{key}".encode()).hexdigest() + comparison_store.put_column( + comparison_key, + series, + declared_dtype=str(metadata["declared_dtype"]), + entity_ids=series.index, + node_key=receipt.key, + ) + copied = comparison_store.metadata(comparison_key, kind="column") + duplicated_coordinate_payload += sum( + int(entry["size"]) for entry in copied["payloads"].values() + ) + else: + ordinary_patch_payload += payload + assert structural_frame_payload > 0 + assert structural_reference_payload == 0 + assert duplicated_coordinate_payload > structural_reference_payload + assert ordinary_patch_payload > 0 + def test_the_parity_fixtures_are_declared_but_not_faked() -> None: """Green from the first commit: no parity fixture is invented here. diff --git a/packages/microcosm-graph/tests/test_artifact_edges.py b/packages/microcosm-graph/tests/test_artifact_edges.py index 0626d6d58..764af22a8 100644 --- a/packages/microcosm-graph/tests/test_artifact_edges.py +++ b/packages/microcosm-graph/tests/test_artifact_edges.py @@ -143,7 +143,7 @@ def test_cross_population_cold_warm_and_roundtrip(tmp_path): restored = RunManifest.from_json(warm.to_json()) assert restored.key == warm.key assert restored.to_json() == warm.to_json() - assert json.loads(warm.to_json())["schema_version"] == 3 + assert json.loads(warm.to_json())["schema_version"] == 4 @pytest.mark.parametrize( diff --git a/packages/microcosm-graph/tests/test_graph_canonical.py b/packages/microcosm-graph/tests/test_graph_canonical.py index ab356959b..6d33a0897 100644 --- a/packages/microcosm-graph/tests/test_graph_canonical.py +++ b/packages/microcosm-graph/tests/test_graph_canonical.py @@ -62,4 +62,7 @@ def test_normative_strips_only_declaration_descriptive_fields() -> None: assert normative(SourceRef("survey", "csv-tables", "human words")) == { "name": "survey", "codec": "csv-tables", + "content_type": "application/octet-stream", + "access": None, + "expected": (), } diff --git a/packages/microcosm-graph/tests/test_graph_decl.py b/packages/microcosm-graph/tests/test_graph_decl.py index 9ee8426ac..2376888f5 100644 --- a/packages/microcosm-graph/tests/test_graph_decl.py +++ b/packages/microcosm-graph/tests/test_graph_decl.py @@ -8,6 +8,9 @@ from __future__ import annotations +import hashlib +from pathlib import Path + import pytest from microcosm.graph import ( @@ -349,3 +352,18 @@ def test_every_declared_name_channel_refuses_dots() -> None: WeightTransition("house.hold", "design", "importance") with pytest.raises(GraphError, match="may not contain '.'"): Graph("toy", (), (), mass_partition=("person", "per.iod")) + + +def test_interface_lock_matches_declaration_and_kernel_modules() -> None: + root = Path(__file__).parents[3] + expected = { + name: digest + for line in (root / "docs/graph-interface.lock").read_text().splitlines() + for digest, name in (line.split(),) + } + graph_package = root / "packages/microcosm-graph/src/microcosm/graph" + actual = { + name: hashlib.sha256((graph_package / name).read_bytes()).hexdigest() + for name in ("decl.py", "kernel.py") + } + assert actual == expected diff --git a/packages/microcosm-graph/tests/test_graph_explain.py b/packages/microcosm-graph/tests/test_graph_explain.py index da74a5385..1985b3ac8 100644 --- a/packages/microcosm-graph/tests/test_graph_explain.py +++ b/packages/microcosm-graph/tests/test_graph_explain.py @@ -13,7 +13,11 @@ import pytest import microcosm.graph as graph_api -from microcosm.graph import describe, explain_html, graph_to_json +from microcosm.graph import ( + describe, + explain_html, + graph_document_to_json, +) ROOT = Path(__file__).parents[3] @@ -431,7 +435,7 @@ def test_saved_run_cli_validates_store_and_reattaches_frames(tmp_path: Path) -> graph_path = tmp_path / "run" / "graph.json" output_path = tmp_path / "rendered.html" run.manifest.save(manifest_path) - graph_path.write_text(graph_to_json(run.compiled.graph), encoding="utf-8") + graph_path.write_text(graph_document_to_json(run.compiled.graph), encoding="utf-8") tool = _tool("graph_explain") tool.render_saved_run(manifest_path, graph_path, output_path) diff --git a/packages/microcosm-graph/tests/test_graph_kernel_contract.py b/packages/microcosm-graph/tests/test_graph_kernel_contract.py index eaff71843..eb9074ba5 100644 --- a/packages/microcosm-graph/tests/test_graph_kernel_contract.py +++ b/packages/microcosm-graph/tests/test_graph_kernel_contract.py @@ -193,7 +193,7 @@ def test_numeric_scope_validates_class_tolerance_and_platform() -> None: def test_context_numerics_default_empty_and_carry_scopes() -> None: """Amendment 17: ``numerics`` defaults empty and rides at the end of the context.""" fields = [f.name for f in dataclasses.fields(KernelContext)] - assert fields[-3:] == ["tolerances", "numerics", "artifacts"] + assert fields[-4:] == ["tolerances", "numerics", "artifacts", "weight_anchors"] scope = NumericScope( numeric=Numeric.PLATFORM_BITWISE, platform="arm64/darwin/py3.13" ) diff --git a/packages/microcosm-graph/tests/test_graph_reconstruction_materialization.py b/packages/microcosm-graph/tests/test_graph_reconstruction_materialization.py new file mode 100644 index 000000000..76c78e4b2 --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_reconstruction_materialization.py @@ -0,0 +1,294 @@ +"""Portable population reconstruction and post-run local materialization.""" + +from __future__ import annotations + +import importlib.util +import json +import shutil +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.frame import Frame +from microcosm.graph import ( + ContentStore, + Graph, + GraphRuntimeError, + MaterializerRegistry, + Product, + ProductKind, + StoreCorruptError, + materialize_products, + reconstruct_population, + run_graph_source, +) + +if "_reconstruction_toy" not in sys.modules: + _SPEC = importlib.util.spec_from_file_location( + "_reconstruction_toy", Path(__file__).with_name("_toy.py") + ) + sys.modules["_reconstruction_toy"] = importlib.util.module_from_spec(_SPEC) + _SPEC.loader.exec_module(sys.modules["_reconstruction_toy"]) +toy = sys.modules["_reconstruction_toy"] + + +def _assert_frame_equal(actual: Frame, expected: Frame) -> None: + assert actual.schema == expected.schema + assert actual.metadata == expected.metadata + assert actual.links == expected.links + for entity in expected.entities: + pd.testing.assert_frame_equal( + actual.table(entity), expected.table(entity), check_exact=True + ) + pd.testing.assert_series_equal(actual.strata, expected.strata, check_exact=True) + assert actual.weighted_entities == expected.weighted_entities + for entity in expected.weighted_entities: + assert actual.weights_for(entity).kind is expected.weights_for(entity).kind + np.testing.assert_array_equal( + actual.weights_for(entity).values, + expected.weights_for(entity).values, + ) + + +def _product_graph() -> Graph: + base = toy.small_graph() + return Graph( + base.country, + base.sources, + base.nodes, + products=( + Product("final.population", ProductKind.POPULATION, node="target_b"), + Product( + "final.h5", + ProductKind.EXPORT, + source="final.population", + codec="test-h5", + codec_version=1, + ), + ), + ) + + +def test_named_population_reconstructs_ordinary_patches_without_runtime_inputs( + tmp_path: Path, +) -> None: + graph = _product_graph() + run = toy.run_toy(graph, tmp_path / "run") + expected = run.manifest.population("survey") + manifest_path = tmp_path / "manifest.json" + run.manifest.save(manifest_path) + shutil.rmtree(next(iter(run.sources.values()))) + + reconstructed = reconstruct_population( + graph, + type(run.manifest).from_json(manifest_path.read_text()), + run.store, + "final.population", + ) + + _assert_frame_equal(reconstructed, expected) + assert "target_a" in reconstructed.table("person") + assert "target_b" in reconstructed.table("person") + + +def test_saved_manifest_rejects_a_different_graph(tmp_path: Path) -> None: + graph = _product_graph() + run = toy.run_toy(graph, tmp_path / "run") + manifest_path = tmp_path / "manifest.json" + run.manifest.save(manifest_path) + different = Graph( + "different-country", + graph.sources, + graph.nodes, + graph.mass_partition, + graph.products, + ) + + with pytest.raises(GraphRuntimeError, match="different Graph"): + reconstruct_population( + different, + run.manifest, + run.store, + "final.population", + ) + with pytest.raises(GraphRuntimeError, match="different Graph"): + materialize_products( + different, + manifest_path, + run.store, + tmp_path / "candidate", + MaterializerRegistry(), + {"final.h5": "microdata.h5"}, + ) + + +def test_structural_coordinates_reference_one_verified_frame_object( + tmp_path: Path, +) -> None: + graph = Graph( + "toy", + (toy.SOURCE,), + (toy.CREATE,), + products=(Product("initial", ProductKind.POPULATION, node="survey"),), + ) + run = toy.run_toy(graph, tmp_path / "run") + receipt = run.manifest.nodes["survey"] + assert receipt.frame_key is not None + frame_metadata = run.store.metadata(receipt.frame_key, kind="frame") + frame_payload_bytes = sum( + int(entry["size"]) for entry in frame_metadata["payloads"].values() + ) + assert frame_payload_bytes > 0 + + for coordinate, key in receipt.artifacts.items(): + metadata = run.store.metadata(key, kind="column") + assert metadata["encoding"] == "frame-column-ref-v1" + assert metadata["frame_key"] == receipt.frame_key + assert metadata["payloads"] == {} + entity, column = coordinate + expected = run.manifest.population("survey").table(entity) + id_column = run.manifest.population("survey").schema.entity_id_column(entity) + pd.testing.assert_series_equal( + run.store.load_column(key), + pd.Series( + expected[column].array.copy(), + index=pd.Index(expected[id_column].array.copy(), name=id_column), + name=column, + dtype=expected[column].dtype, + ), + check_exact=True, + ) + + altered_key = next(iter(receipt.artifacts.values())) + metadata_path = run.store.object_path(altered_key) / "meta.json" + altered = json.loads(metadata_path.read_text()) + altered["column"] = "different_column" + metadata_path.write_text(json.dumps(altered)) + with pytest.raises(StoreCorruptError, match="node identity"): + run.store.load_column(altered_key) + + +def test_h5_materialization_uses_saved_values_and_records_candidate_identity( + tmp_path: Path, +) -> None: + graph = _product_graph() + run = toy.run_toy(graph, tmp_path / "run") + manifest_path = tmp_path / "manifest.json" + run.manifest.save(manifest_path) + calls_before = toy.total_calls(run.registry) + shutil.rmtree(next(iter(run.sources.values()))) + + registry = MaterializerRegistry() + + def write_h5(value: object, destination: Path) -> None: + import h5py + + assert isinstance(value, Frame) + person = value.table("person") + with h5py.File(destination, mode="w") as h5: + h5.create_dataset( + "person_id", + data=person["person_id"].to_numpy(), + track_times=False, + ) + h5.create_dataset( + "target_b", + data=person["target_b"].to_numpy(), + track_times=False, + ) + + registry.register("test-h5", 1, write_h5, implementation_hash="a" * 64) + first = materialize_products( + graph, + manifest_path, + run.store, + tmp_path / "candidate-one", + registry, + {"final.h5": "microdata.h5"}, + ) + second = materialize_products( + graph, + manifest_path, + run.store, + tmp_path / "candidate-two", + registry, + {"final.h5": "microdata.h5"}, + ) + + assert toy.total_calls(run.registry) == calls_before + assert first.key == second.key + assert first.to_json() == second.to_json() + record = first.products["final.h5"] + assert record.boundary == "file" + assert record.size > 0 + assert len(record.sha256) == 64 + assert len(record.content_key) == 64 + saved = json.loads((tmp_path / "candidate-one/candidate-index.json").read_text()) + assert saved["manifest_key"] == run.manifest.key + assert saved["products"]["final.h5"]["sha256"] == record.sha256 + + +def test_shared_runner_executes_one_yaml_root_and_saves_before_materializing( + tmp_path: Path, +) -> None: + graph_yaml = tmp_path / "graph.yaml" + graph_yaml.write_text( + """\ +schema_version: 1 +country: toy +sources: + - {name: survey, codec: csv-tables} +nodes: + - id: survey + kernel: source.csv@1 + structural: create + sources: [survey] + outputs: + - {entity: person, column: age, dtype: int64} + - {entity: person, column: income, dtype: float64} + - {entity: person, column: is_adult, dtype: boolean} + - {entity: person, column: receives_x, dtype: boolean} + - {entity: household, column: household_size, dtype: int64} +products: + - {name: final.population, kind: population, target: {node: survey}} + - name: final.h5 + kind: export + target: {product: final.population} + codec: test-h5 + codec_version: 1 +""" + ) + sources = toy.toy_sources(tmp_path / "sources") + kernels = toy.toy_registry() + manifest_path = tmp_path / "evidence/run.json" + registry = MaterializerRegistry() + + def write_after_manifest(value: object, destination: Path) -> None: + assert manifest_path.is_file() + assert isinstance(value, Frame) + destination.write_bytes( + b"test-h5\0" + value.table("person").to_csv(index=False).encode() + ) + + registry.register("test-h5", 1, write_after_manifest, implementation_hash="b" * 64) + result = run_graph_source( + graph_yaml, + sources=sources, + store=ContentStore(tmp_path / "store"), + kernels=kernels, + manifest_path=manifest_path, + graph_json_path=tmp_path / "evidence/graph.json", + materializers=registry, + candidate_directory=tmp_path / "candidate", + materialized_outputs={"final.h5": "microdata.h5"}, + ) + + assert toy.total_calls(kernels) == 1 + assert result.manifest_path == manifest_path + assert result.candidate_index is not None + assert result.candidate_index.manifest_key == result.manifest.key + assert (tmp_path / "candidate/candidate-index.json").is_file() + assert (tmp_path / "evidence/graph.json").is_file() diff --git a/packages/microcosm-graph/tests/test_graph_run_contracts.py b/packages/microcosm-graph/tests/test_graph_run_contracts.py new file mode 100644 index 000000000..3bb982c7d --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_run_contracts.py @@ -0,0 +1,322 @@ +"""Verified source bindings and run-level validation behavior.""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import sys +from dataclasses import replace +from pathlib import Path +from types import MappingProxyType + +import pandas as pd +import pytest + +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + ExpectedContent, + Graph, + GraphSourceReceipt, + KernelBase, + KernelResult, + LoadedGraphSource, + Node, + Owned, + Product, + ProductKind, + RunManifest, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + graph_document_to_json, + run_graph, +) +from microcosm.graph.codecs import SourceCodecRegistry, load_csv_tables, load_source +from microcosm.graph.store import StoreUnavailable + +spec = importlib.util.spec_from_file_location( + "_run_contract_toy", Path(__file__).with_name("_toy.py") +) +toy = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = toy +spec.loader.exec_module(toy) + + +class WrongCodec(KernelBase): + ref = "source.wrong-codec@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): + return KernelResult(frame=load_source("frame-store", context.sources["survey"])) + + +class RecordAdvisory(KernelBase): + ref = "test.record-advisory@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + table = context.tables["release"] + assert table["gate_verdict"].iloc[0] == "fail" + ids = pd.Index(table["release_id"], name="release_id") + return KernelResult( + columns={ + ("release", "advisory_seen"): pd.Series( + [True], index=ids, dtype="boolean" + ) + } + ) + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_expected_source_identity_is_verified_and_recorded(tmp_path: Path) -> None: + sources = toy.toy_sources(tmp_path) + member = sources["survey"] / "person.csv" + source = SourceRef( + "survey", + "csv-tables", + content_type="application/vnd.microcosm.frame-source", + access="public", + expected=( + ExpectedContent( + _sha256(member), + boundary="member", + path="person.csv", + size=member.stat().st_size, + identity_ref="chronicle://census/example/2023/person.csv", + ), + ), + ) + manifest = run_graph( + compile_graph(Graph("toy", (source,), (toy.CREATE,))), + sources=sources, + store=ContentStore(tmp_path / "store"), + kernels=toy.toy_registry(), + ) + + binding = manifest.source_bindings["survey"] + assert binding["codec"] == "csv-tables" + assert binding["content_type"] == "application/vnd.microcosm.frame-source" + assert binding["access"] == "public" + assert binding["expected"][0]["matched"] is True + assert binding["expected"][0]["calculated_sha256"] == _sha256(member) + assert len(binding["codec_impl_hash"]) == 64 + assert manifest.graph_key + assert json.loads(manifest.to_json())["schema_version"] == 4 + assert RunManifest.from_json(manifest.to_json()).graph_key == manifest.graph_key + + +def test_manifest_binds_yaml_parameters_and_optional_graph_json( + tmp_path: Path, +) -> None: + graph = Graph("toy", (toy.SOURCE,), (toy.CREATE,)) + loaded = LoadedGraphSource( + graph=graph, + schema_version=1, + parameters=MappingProxyType({"period": 2024}), + receipts=(GraphSourceReceipt("graph.yaml", "a" * 64),), + ) + graph_json = graph_document_to_json(graph) + store = ContentStore(tmp_path / "store") + manifest = run_graph( + compile_graph(graph), + sources=toy.toy_sources(tmp_path), + store=store, + kernels=toy.toy_registry(), + graph_source=loaded, + graph_json=graph_json, + ) + + assert manifest.graph_source_receipts == ( + {"path": "graph.yaml", "sha256": "a" * 64}, + ) + assert manifest.parameters == {"period": 2024} + assert manifest.graph_json_key is not None + assert store.load_bytes(manifest.graph_json_key) == graph_json.encode() + + mismatched = replace(loaded, graph=replace(graph, country="other")) + with pytest.raises(ValueError, match="does not describe"): + run_graph( + compile_graph(graph), + sources=toy.toy_sources(tmp_path, name="other-source"), + store=store, + kernels=toy.toy_registry(), + graph_source=mismatched, + ) + + +def test_source_identity_mismatch_prevents_kernel_execution(tmp_path: Path) -> None: + registry = toy.toy_registry() + source_kernel = registry.get(toy.CREATE.kernel) + source = replace( + toy.SOURCE, + expected=(ExpectedContent("0" * 64, boundary="member", path="person.csv"),), + ) + with pytest.raises(StoreUnavailable, match="content identity mismatch"): + run_graph( + compile_graph(Graph("toy", (source,), (toy.CREATE,))), + sources=toy.toy_sources(tmp_path), + store=ContentStore(tmp_path / "store"), + kernels=registry, + ) + assert source_kernel.calls == 0 + + +def test_codec_name_and_implementation_change_consuming_identity( + tmp_path: Path, +) -> None: + sources = toy.toy_sources(tmp_path) + first_codecs = SourceCodecRegistry() + first_codecs.register("csv-copy", load_csv_tables, implementation_hash="1" * 64) + second_codecs = SourceCodecRegistry() + second_codecs.register("csv-copy", load_csv_tables, implementation_hash="2" * 64) + changed_name = SourceCodecRegistry() + changed_name.register("csv-other", load_csv_tables, implementation_hash="1" * 64) + + def execute(codec: str, codecs: SourceCodecRegistry, suffix: str) -> str: + graph = Graph("toy", (SourceRef("survey", codec),), (toy.CREATE,)) + return ( + run_graph( + compile_graph(graph), + sources=sources, + store=ContentStore(tmp_path / suffix, codecs=codecs), + kernels=toy.toy_registry(), + ) + .nodes["survey"] + .key + ) + + first = execute("csv-copy", first_codecs, "first") + assert execute("csv-copy", second_codecs, "second") != first + assert execute("csv-other", changed_name, "third") != first + + +def test_kernel_cannot_override_declared_source_codec(tmp_path: Path) -> None: + registry = toy.toy_registry() + registry.register(WrongCodec()) + graph = Graph( + "toy", + (toy.SOURCE,), + (replace(toy.CREATE, kernel=WrongCodec.ref),), + ) + with pytest.raises(StoreUnavailable, match="declares codec 'csv-tables'"): + run_graph( + compile_graph(graph), + sources=toy.toy_sources(tmp_path), + store=ContentStore(tmp_path / "store"), + kernels=registry, + ) + + +def test_failed_required_validation_records_unreached_descendants( + tmp_path: Path, +) -> None: + gate = toy.gate_node( + "income_validation", + population="survey", + column="income", + low=-2.0, + high=-1.0, + ) + blocked = replace( + toy.derive("after_validation", ("age",), "after_validation"), + requires_success=("income.valid",), + ) + graph = Graph( + "toy", + (toy.SOURCE,), + (blocked, gate, toy.CREATE), + products=( + Product("income.valid", ProductKind.VALIDATION, node="income_validation"), + Product( + "after.population", ProductKind.POPULATION, node="after_validation" + ), + ), + ) + compiled = compile_graph(graph) + store = ContentStore(tmp_path / "store") + sources = toy.toy_sources(tmp_path) + registry = toy.toy_registry() + manifest = run_graph(compiled, sources=sources, store=store, kernels=registry) + + validation = manifest.nodes["income_validation"] + descendant = manifest.nodes["after_validation"] + assert validation.receipt["outcome"] == "fail" + assert validation.outcome_key is not None + assert ( + store.load_json(validation.outcome_key, kind="validation-outcome")["outcome"] + == "fail" + ) + assert descendant.status == "unreached" + assert descendant.blocked_by == ("income_validation",) + assert not descendant.artifacts + assert manifest.outcome == "not_successful" + assert manifest.products["income.valid"]["key"] == validation.outcome_key + assert manifest.products["after.population"]["status"] == "unreached" + + resumed = run_graph( + compiled, + sources=sources, + store=store, + kernels=registry, + resume="require", + ) + assert resumed.nodes["income_validation"].hit + assert resumed.nodes["after_validation"].status == "unreached" + assert RunManifest.from_json(resumed.to_json()).outcome == "not_successful" + + +def test_advisory_validation_is_recorded_without_blocking_dependents( + tmp_path: Path, +) -> None: + advisory = toy.gate_node( + "income_advisory", + population="survey", + column="income", + low=-2.0, + high=-1.0, + ) + dependent = Node( + "record_advisory", + RecordAdvisory.ref, + inputs=(Slice("release", ("gate_verdict",)),), + outputs=(Owned("release", "advisory_seen", "boolean"),), + population="survey", + ) + graph = Graph( + "toy", + (toy.SOURCE,), + (dependent, advisory, toy.CREATE), + products=( + Product("income.advisory", ProductKind.VALIDATION, node=advisory.id), + ), + ) + registry = toy.toy_registry() + registry.register(RecordAdvisory()) + store = ContentStore(tmp_path / "store") + + manifest = run_graph( + compile_graph(graph), + sources=toy.toy_sources(tmp_path), + store=store, + kernels=registry, + ) + + validation = manifest.nodes[advisory.id] + assert validation.receipt["outcome"] == "fail" + assert validation.outcome_key is not None + assert ( + store.load_json(validation.outcome_key, kind="validation-outcome")["outcome"] + == "fail" + ) + assert manifest.nodes[dependent.id].status == "executed" + assert manifest.products["income.advisory"]["key"] == validation.outcome_key + assert manifest.outcome == "success" diff --git a/packages/microcosm-graph/tests/test_graph_serialize.py b/packages/microcosm-graph/tests/test_graph_serialize.py index fe45c0d1a..cd20ec689 100644 --- a/packages/microcosm-graph/tests/test_graph_serialize.py +++ b/packages/microcosm-graph/tests/test_graph_serialize.py @@ -11,6 +11,7 @@ import microcosm.graph as graph_api from microcosm.graph import ( + ExpectedContent, Graph, Node, Owned, @@ -65,7 +66,23 @@ def _graph() -> Graph: ) return Graph( "toy", - (SourceRef("fixture", "csv-tables", description="pinned table"),), + ( + SourceRef( + "fixture", + "csv-tables", + description="pinned table", + content_type="application/vnd.microcosm.frame-source", + access="licensed", + expected=( + ExpectedContent( + "a" * 64, + boundary="member", + path="person.csv", + size=42, + ), + ), + ), + ), (source, absent, pool), ) @@ -89,6 +106,7 @@ def test_graph_json_round_trip_is_lossless_and_canonical() -> None: payload = json.loads(text) assert payload["sources"][0]["codec"] == "csv-tables" + assert payload["sources"][0]["expected"][0]["path"] == "person.csv" assert payload["nodes"][1]["params"]["nested"] == [True, None, [2.5, "x"]] assert list(payload) == ["country", "nodes", "sources"] @@ -105,9 +123,9 @@ def test_graph_from_json_rejects_shape_enum_and_parameter_drift() -> None: graph_from_json(json.dumps(payload)) payload = json.loads(graph_to_json(_graph())) - payload["nodes"][0]["params"]["revision"] = {"not": "a Param"} - with pytest.raises(TypeError, match="legal graph parameter"): - graph_from_json(json.dumps(payload)) + payload["nodes"][0]["params"]["revision"] = {"nested": [1, {"ok": True}]} + restored = graph_from_json(json.dumps(payload)) + assert restored.nodes[0].params["revision"]["nested"] == (1, {"ok": True}) with pytest.raises(ValueError, match="non-finite"): graph_from_json(graph_to_json(_graph()).replace("2.5", "NaN")) diff --git a/packages/microcosm-graph/tests/test_graph_source.py b/packages/microcosm-graph/tests/test_graph_source.py new file mode 100644 index 000000000..8d46a88d5 --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_source.py @@ -0,0 +1,389 @@ +"""Authored graph YAML composes deterministically into one declaration.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Capabilities, + Determinism, + ExpectedContent, + Graph, + GraphParameterBindingError, + GraphSourceCompositionError, + GraphSourceParseError, + GraphSourceValidationError, + KernelBase, + KernelRegistry, + Node, + Owned, + Ownership, + Product, + ProductKind, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, + compiled_graph_from_yaml_file, + graph_document_from_json, + graph_document_to_json, + graph_from_yaml_file, + load_graph_source, + load_yaml12, +) + +SOURCES = """\ +schema_version: 1 +sources: + - name: fixture + codec: csv-tables + description: exact fixture +""" + +NODES = """\ +schema_version: 1 +nodes: + - id: apply + kernel: apply@1 + population: create + inputs: + - entity: person + columns: [age] + outputs: + - entity: person + column: score + dtype: float64 + param_bindings: + fraction: sample_fraction + - id: create + kernel: source.csv@1 + structural: create + sources: [fixture] + outputs: + - entity: person + column: age + dtype: int64 +""" + + +class _Create(KernelBase): + ref = "source.csv@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE + ) + + def run(self, context): # pragma: no cover - compilation does not execute + raise AssertionError("not executed") + + +class _Apply(KernelBase): + ref = "apply@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): # pragma: no cover - compilation does not execute + raise AssertionError("not executed") + + +def _registry() -> KernelRegistry: + registry = KernelRegistry() + registry.register(_Create()) + registry.register(_Apply()) + return registry + + +def _write_graph(root: Path, module_order: tuple[str, str]) -> Path: + root.mkdir() + (root / "sources.yaml").write_text(SOURCES) + (root / "nodes.yaml").write_text(NODES) + graph = root / "graph.yaml" + graph.write_text( + "schema_version: 1\n" + "country: toy\n" + "modules:\n" + + "".join(f" - {name}\n" for name in module_order) + + "parameters:\n" + " sample_fraction:\n" + " type: number\n" + " required: false\n" + " default: 1.0\n" + " minimum: 0.0\n" + " maximum: 1.0\n" + ) + return graph + + +def test_modules_lower_once_independent_of_authored_order(tmp_path: Path) -> None: + first = _write_graph(tmp_path / "first", ("sources.yaml", "nodes.yaml")) + second = _write_graph(tmp_path / "second", ("nodes.yaml", "sources.yaml")) + + one = load_graph_source(first, parameters={"sample_fraction": 0.25}) + two = load_graph_source(second, parameters={"sample_fraction": 0.25}) + + assert one.graph == two.graph + assert tuple(node.id for node in one.graph.nodes) == ("apply", "create") + assert one.graph.node("apply").params["fraction"] == 0.25 + assert compiled_graph_from_yaml_file(first, kernels=_registry()).order == ( + "create", + "apply", + ) + assert tuple(receipt.path for receipt in one.receipts) == ( + "graph.yaml", + "nodes.yaml", + "sources.yaml", + ) + generated = graph_document_to_json(one.graph) + assert graph_document_from_json(generated) == one.graph + assert json.loads(generated)["derived"] is True + + +def test_parameter_binding_is_declared_and_affects_only_consumers( + tmp_path: Path, +) -> None: + path = _write_graph(tmp_path / "graph", ("sources.yaml", "nodes.yaml")) + default = graph_from_yaml_file(path) + changed = graph_from_yaml_file(path, parameters={"sample_fraction": 0.5}) + assert default.node("create") == changed.node("create") + assert default.node("apply") != changed.node("apply") + with pytest.raises(GraphParameterBindingError, match="undeclared"): + graph_from_yaml_file(path, parameters={"node.kernel": "other@1"}) + + +def test_registry_contract_is_checked_before_execution(tmp_path: Path) -> None: + path = _write_graph(tmp_path / "graph", ("sources.yaml", "nodes.yaml")) + registry = _registry() + registry.get("source.csv@1").capabilities = Capabilities( # type: ignore[assignment] + Determinism.DETERMINISTIC + ) + with pytest.raises(GraphSourceValidationError, match="structural operation"): + compiled_graph_from_yaml_file(path, kernels=registry) + + +def test_every_stack_base_declaration_field_lowers_exactly(tmp_path: Path) -> None: + model = ArtifactType("test.model", 1) + path = tmp_path / "graph.yaml" + path.write_text( + """\ +schema_version: 1 +country: toy +mass_partition: [person, period] +products: + - {name: final, kind: population, target: {node: weights}} +sources: + - name: fixture + codec: csv-tables + content_type: application/vnd.microcosm.frame-source + access: licensed + description: exact fixture + expected: + - {sha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, boundary: member, path: person.csv, size: 42, identity_ref: "chronicle://example"} +nodes: + - id: create + kernel: create@1 + structural: create + sources: [fixture] + params: {nested: [true, null, [2.5, x]]} + outputs: + - {entity: person, column: age, dtype: int64} + - {entity: person, column: period, dtype: int64} + - {entity: person, column: selected, dtype: boolean} + description: source node + citation: synthetic + - id: train + kernel: train@1 + population: create + inputs: + - {entity: person, columns: [age]} + artifact_outputs: + - {name: model, type: {name: test.model, schema_version: 1}} + - id: expand + kernel: expand@1 + structural: expand + base: create + inputs: + - {entity: person, columns: [age]} + entrants: true + mass: free + - id: apply + kernel: apply@1 + population: expand + inputs: + - {entity: person, columns: [age, selected], rows: selected} + outputs: + - {entity: person, column: age, dtype: int64, rows: selected, ownership: produced, rewrite: true} + - {entity: person, column: score, dtype: float64, rows: selected, ownership: absent} + artifact_inputs: + - {name: fitted, producer: train, artifact: model, type: {name: test.model, schema_version: 1}} + - id: weights + kernel: weights@1 + structural: reweight + base: expand + inputs: + - {entity: person, columns: [age]} + weights: {entity: person, to_kind: importance, mass: free} + mass: free +""" + ) + expected = Graph( + "toy", + ( + SourceRef( + "fixture", + "csv-tables", + "exact fixture", + "application/vnd.microcosm.frame-source", + "licensed", + ( + ExpectedContent( + "a" * 64, + "member", + "person.csv", + 42, + "chronicle://example", + ), + ), + ), + ), + ( + Node( + "apply", + "apply@1", + inputs=(Slice("person", ("age", "selected"), "selected"),), + outputs=( + Owned("person", "age", "int64", "selected", rewrite=True), + Owned( + "person", + "score", + "float64", + "selected", + Ownership.ABSENT, + ), + ), + population="expand", + artifact_inputs=(ArtifactInput("fitted", "train", "model", model),), + ), + Node( + "create", + "create@1", + outputs=( + Owned("person", "age", "int64"), + Owned("person", "period", "int64"), + Owned("person", "selected", "boolean"), + ), + params={"nested": (True, None, (2.5, "x"))}, + structural=StructuralDelta.CREATE, + sources=("fixture",), + description="source node", + citation="synthetic", + ), + Node( + "expand", + "expand@1", + inputs=(Slice("person", ("age",)),), + structural=StructuralDelta.EXPAND, + base="create", + mass="free", + entrants=True, + ), + Node( + "train", + "train@1", + inputs=(Slice("person", ("age",)),), + population="create", + artifact_outputs=(ArtifactOutput("model", model),), + ), + Node( + "weights", + "weights@1", + inputs=(Slice("person", ("age",)),), + structural=StructuralDelta.REWEIGHT, + base="expand", + weights=WeightTransition("person", "importance", "free"), + mass="free", + ), + ), + ("person", "period"), + products=(Product("final", ProductKind.POPULATION, node="weights"),), + ) + assert graph_from_yaml_file(path) == expected + + +def test_closed_schema_reports_pointer_and_source_location(tmp_path: Path) -> None: + path = _write_graph(tmp_path / "graph", ("sources.yaml", "nodes.yaml")) + nodes = path.parent / "nodes.yaml" + nodes.write_text( + NODES.replace(" kernel: apply@1", " kernel: apply@1\n surprise: true") + ) + with pytest.raises(GraphSourceValidationError) as caught: + graph_from_yaml_file(path) + assert caught.value.source == str(nodes) + assert caught.value.pointer == "/nodes/0/surprise" + assert (caught.value.line, caught.value.column) == (5, 5) + + +def test_unsupported_graph_source_version_is_rejected(tmp_path: Path) -> None: + path = _write_graph(tmp_path / "graph", ("sources.yaml", "nodes.yaml")) + path.write_text(path.read_text().replace("schema_version: 1", "schema_version: 2")) + with pytest.raises(GraphSourceValidationError, match="schema_version"): + graph_from_yaml_file(path) + + +@pytest.mark.parametrize( + "bad", + ["../outside.yaml", "/absolute.yaml", "directory\\module.yaml"], +) +def test_unsafe_module_paths_are_rejected(tmp_path: Path, bad: str) -> None: + graph = tmp_path / "graph.yaml" + graph.write_text(f"schema_version: 1\ncountry: toy\nmodules: [{bad!r}]\n") + with pytest.raises(GraphSourceCompositionError, match="module path|unsafe"): + graph_from_yaml_file(graph) + + +def test_duplicate_and_recursive_modules_are_rejected(tmp_path: Path) -> None: + (tmp_path / "module.yaml").write_text("schema_version: 1\nmodules: [module.yaml]\n") + graph = tmp_path / "graph.yaml" + graph.write_text("schema_version: 1\ncountry: toy\nmodules: [module.yaml]\n") + with pytest.raises(GraphSourceCompositionError, match="cycle"): + graph_from_yaml_file(graph) + + +@pytest.mark.parametrize( + "text", + [ + "value: 1\nvalue: 2\n", + "value: !!str 1\n", + "value: 2026-09-08\n", + "value: .nan\n", + "first: 1\n---\nsecond: 2\n", + "1: value\n", + "cycle: &cycle [*cycle]\n", + ], +) +def test_shared_yaml_parser_rejects_ambiguous_features(text: str) -> None: + with pytest.raises(GraphSourceParseError): + load_yaml12(text, source="fixture.yaml") + + +def test_generated_graph_document_has_a_closed_versioned_envelope( + tmp_path: Path, +) -> None: + path = _write_graph(tmp_path / "graph", ("sources.yaml", "nodes.yaml")) + generated = graph_document_to_json(graph_from_yaml_file(path)) + payload = json.loads(generated) + payload["serialization_version"] = 2 + with pytest.raises(ValueError, match="Unsupported"): + graph_document_from_json(json.dumps(payload)) + payload["serialization_version"] = 1 + payload["extra"] = True + with pytest.raises(ValueError, match="fields"): + graph_document_from_json(json.dumps(payload)) + with pytest.raises(ValueError, match="duplicate"): + graph_document_from_json( + '{"derived":true,"derived":true,"document_type":"microcosm.graph",' + '"graph":{},"serialization_version":1}' + ) diff --git a/packages/microcosm-graph/tests/test_graph_state.py b/packages/microcosm-graph/tests/test_graph_state.py new file mode 100644 index 000000000..de709e795 --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_state.py @@ -0,0 +1,318 @@ +"""Population-state operations retain values, lineage, products, and anchors.""" + +from __future__ import annotations + +import importlib.util +import sys +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from microcosm.frame import Frame, WeightKind, Weights +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelResult, + Node, + Owned, + Population, + PopulationError, + Product, + ProductKind, + Slice, + StructuralDelta, + WeightTransition, + compile_graph, + graph_from_json, + graph_to_json, + run_graph, +) +from microcosm.graph.population import patch, union_populations + +spec = importlib.util.spec_from_file_location( + "_state_toy", Path(__file__).with_name("_toy.py") +) +toy = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = toy +spec.loader.exec_module(toy) + + +class ReviseIncome(KernelBase): + ref = "state.revise-income@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REVISION + ) + + def run(self, context): + table = context.tables["person"] + ids = pd.Index(table.person_id, name="person_id") + values = table.income.to_numpy() + float(context.params["increment"]) + return KernelResult( + columns={ + ("person", "income"): pd.Series(values, index=ids, dtype="float64") + } + ) + + +class Union(KernelBase): + ref = "graph.union@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.UNION + ) + + def run(self, context): + assert not context.tables + return KernelResult() + + +class AnchorCalibration(KernelBase): + ref = "state.anchor-calibration@1" + capabilities = Capabilities( + Determinism.DETERMINISTIC, structural=StructuralDelta.REWEIGHT + ) + + def run(self, context): + current = context.weights["household"] + anchor = context.weight_anchors["pool.weights"] + assert np.array_equal(current.values, anchor.values) + return KernelResult(weights=Weights(current.values, WeightKind.CALIBRATED)) + + +def test_successive_value_revisions_preserve_each_value_artifact( + tmp_path: Path, +) -> None: + first = Node( + "revision.one", + ReviseIncome.ref, + inputs=(Slice("person", ("income",)),), + outputs=(Owned("person", "income", "float64", rewrite=True),), + params={"increment": 1.0}, + structural=StructuralDelta.REVISION, + base="survey", + ) + second = replace(first, id="revision.two", base=first.id, params={"increment": 2.0}) + graph = Graph("toy", (toy.SOURCE,), (toy.CREATE, first, second)) + compiled = compile_graph(graph) + registry = toy.toy_registry() + registry.register(ReviseIncome()) + store = ContentStore(tmp_path / "store") + sources = toy.toy_sources(tmp_path) + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=registry, + ) + + source_key = manifest.nodes["survey"].artifacts[("person", "income")] + first_key = manifest.nodes[first.id].artifacts[("person", "income")] + second_key = manifest.nodes[second.id].artifacts[("person", "income")] + original = store.load_column(source_key) + after_first = store.load_column(first_key) + after_second = store.load_column(second_key) + assert np.array_equal(after_first.to_numpy(), original.to_numpy() + 1.0) + assert np.array_equal(after_second.to_numpy(), original.to_numpy() + 3.0) + assert manifest.nodes[first.id].frame_key is None + assert manifest.nodes[second.id].frame_key is None + assert compiled.owners[(second.id, "person", "income")] == second.id + + warm = run_graph( + compiled, + sources=sources, + store=store, + kernels=registry, + ) + assert warm.nodes[first.id].hit and warm.nodes[second.id].hit + + +def test_nested_parameters_are_recursively_frozen_and_lossless() -> None: + nested = {"solver": {"steps": [1, 2], "options": {"exact": True}}} + node = replace(toy.CREATE, params=nested) + nested["solver"]["steps"].append(3) + assert node.params["solver"]["steps"] == (1, 2) + with pytest.raises(TypeError): + node.params["solver"]["new"] = "value" + restored = graph_from_json(graph_to_json(Graph("toy", (toy.SOURCE,), (node,)))) + assert restored == Graph("toy", (toy.SOURCE,), (node,)) + + +def test_products_reject_missing_and_incompatible_targets() -> None: + missing = Graph( + "toy", + (toy.SOURCE,), + (toy.CREATE,), + products=(Product("missing", ProductKind.POPULATION, node="unknown"),), + ) + with pytest.raises(ValueError, match="unknown node"): + compile_graph(missing) + wrong_artifact = Graph( + "toy", + (toy.SOURCE,), + (toy.CREATE,), + products=( + Product( + "model", + ProductKind.ARTIFACT, + node="survey", + artifact="absent", + ), + ), + ) + with pytest.raises(ValueError, match="undeclared artifact"): + compile_graph(wrong_artifact) + + +def test_union_remaps_collisions_and_records_source_lineage(tmp_path: Path) -> None: + asec = replace(toy.CREATE, id="asec") + acs = replace(toy.CREATE, id="acs") + union = Node( + "combined", + Union.ref, + structural=StructuralDelta.UNION, + bases=("asec", "acs"), + ) + graph = Graph( + "toy", + (toy.SOURCE,), + (union, acs, asec), + products=( + Product("combined.population", ProductKind.POPULATION, node="combined"), + ), + ) + compiled = compile_graph(graph) + registry = toy.toy_registry() + registry.register(Union()) + store = ContentStore(tmp_path / "store") + manifest = run_graph( + compiled, + sources=toy.toy_sources(tmp_path), + store=store, + kernels=registry, + ) + + source = manifest.populations["asec"] + combined = manifest.populations["combined"] + assert combined.n("person") == 2 * source.n("person") + assert combined.table("person").person_id.is_unique + assert combined.table("household").household_id.is_unique + assert compiled.product_nodes["combined.population"] == "combined" + lineage_ref = manifest.nodes["combined"].receipt["union_lineage"] + lineage = store.load_json(lineage_ref["key"], kind="union-lineage") + assert len(lineage["person"]) == combined.n("person") + assert {entry[1] for entry in lineage["person"]} == {"acs", "asec"} + assert manifest.mass_ledgers["combined"][-1].operation == "union" + + +def test_union_rejects_incompatible_weight_kinds(tmp_path: Path) -> None: + source = toy.read_toy_frame(toy.toy_sources(tmp_path)["survey"]) + source_weights = source.weights_for("household") + incompatible = Frame( + {entity: source.table(entity).copy() for entity in source.entities}, + source.schema, + { + "household": Weights( + source_weights.values.copy(), + WeightKind.CALIBRATED, + ) + }, + source.strata, + metadata=source.metadata, + ) + union = Node( + "combined", + Union.ref, + structural=StructuralDelta.UNION, + bases=("asec", "acs"), + mass="free", + ) + + with pytest.raises(PopulationError, match="incompatible.*weight kind"): + union_populations( + { + "asec": Population.from_frame(source, "asec"), + "acs": Population.from_frame(incompatible, "acs"), + }, + union, + ) + + +def test_named_weight_anchor_is_aligned_exposed_and_recorded(tmp_path: Path) -> None: + calibration = Node( + "calibrated", + AnchorCalibration.ref, + inputs=( + Slice("person", ("age",)), + Slice("household", ("household_size",)), + ), + params={"max_weight_ratio": 1.0}, + structural=StructuralDelta.REWEIGHT, + base="pool", + weights=WeightTransition("household", "calibrated", "conserve", "pool.weights"), + mass="conserve", + ) + graph = Graph( + "toy", + (toy.SOURCE,), + (toy.CREATE, toy.POOL, calibration), + products=( + Product( + "pool.weights", + ProductKind.WEIGHTS, + node="pool", + entity="household", + ), + Product("final", ProductKind.POPULATION, node="calibrated"), + ), + ) + compiled = compile_graph(graph) + registry = toy.toy_registry() + registry.register(AnchorCalibration()) + manifest = run_graph( + compiled, + sources=toy.toy_sources(tmp_path), + store=ContentStore(tmp_path / "store"), + kernels=registry, + ) + receipt = manifest.nodes["calibrated"].receipt["weight_anchor"] + assert receipt["product"] == "pool.weights" + assert receipt["producer"] == "pool" + assert receipt["producer_key"] == manifest.nodes["pool"].key + assert receipt["realized_max_weight_ratio"] == 1.0 + + +def test_named_weight_anchor_controls_mass_comparison(tmp_path: Path) -> None: + source = toy.read_toy_frame(toy.toy_sources(tmp_path)["survey"]) + source_weights = source.weights_for("household") + incumbent = Frame( + {entity: source.table(entity).copy() for entity in source.entities}, + source.schema, + {"household": Weights(source_weights.values * 2.0, WeightKind.IMPORTANCE)}, + source.strata, + metadata=source.metadata, + ) + population = Population.from_frame(incumbent, "importance") + node = Node( + "calibrated", + "test@1", + structural=StructuralDelta.REWEIGHT, + base="importance", + weights=WeightTransition( + "household", "calibrated", "conserve", "reviewed.weights" + ), + mass="conserve", + ) + result = patch( + population, + node, + KernelResult(weights=Weights(source_weights.values, WeightKind.CALIBRATED)), + weight_anchor=source_weights, + ) + record = result.mass_ledger[-1] + assert np.isclose(record.before_total, record.after_total) + assert np.isclose(record.before_total, float(source.stratum_mass().sum())) diff --git a/tools/graph_explain.py b/tools/graph_explain.py index fb82636e2..c6f57e39f 100644 --- a/tools/graph_explain.py +++ b/tools/graph_explain.py @@ -13,7 +13,10 @@ StructuralDelta, compile_graph, explain_html, + graph_document_from_json, graph_from_json, + graph_key, + load_graph_source, ) @@ -30,6 +33,18 @@ def _load_populations(compiled, manifest: RunManifest, store: ContentStore): return replace(manifest, populations=populations) +def _load_graph(path: Path): + """Load authoritative YAML, generated Graph JSON, or legacy Graph JSON.""" + + if path.suffix.lower() in {".yaml", ".yml"}: + return load_graph_source(path).graph + text = path.read_text(encoding="utf-8") + try: + return graph_document_from_json(text) + except (TypeError, ValueError): + return graph_from_json(text) + + def render_saved_run( manifest_path: Path, graph_path: Path, @@ -39,7 +54,7 @@ def render_saved_run( ) -> None: """Validate and render one saved manifest/graph pair.""" - graph = graph_from_json(graph_path.read_text(encoding="utf-8")) + graph = _load_graph(graph_path) compiled = compile_graph(graph) resolved_store = store_path or manifest_path.parent / "store" if not resolved_store.is_dir(): @@ -49,6 +64,8 @@ def render_saved_run( ) store = ContentStore(resolved_store) manifest = RunManifest.load(manifest_path, store=store) + if manifest.graph_key and manifest.graph_key != graph_key(graph): + raise ValueError("The saved manifest belongs to a different Graph.") manifest = _load_populations(compiled, manifest, store) rendered = explain_html(compiled, manifest) output_path.parent.mkdir(parents=True, exist_ok=True) diff --git a/uv.lock b/uv.lock index b3bfb8926..eccf80f3b 100644 --- a/uv.lock +++ b/uv.lock @@ -949,26 +949,34 @@ name = "microcosm-graph" version = "0.1.0" source = { editable = "packages/microcosm-graph" } dependencies = [ + { name = "jsonschema" }, { name = "microcosm-frame" }, { name = "numpy" }, { name = "pandas" }, + { name = "pyyaml" }, + { name = "referencing" }, ] [package.dev-dependencies] dev = [ + { name = "h5py" }, { name = "hypothesis" }, { name = "pytest" }, ] [package.metadata] requires-dist = [ + { name = "jsonschema", specifier = ">=4.23,<5" }, { name = "microcosm-frame", editable = "packages/microcosm-frame" }, { name = "numpy", specifier = ">=2" }, { name = "pandas", specifier = ">=2.3" }, + { name = "pyyaml", specifier = ">=6" }, + { name = "referencing", specifier = ">=0.35,<1" }, ] [package.metadata.requires-dev] dev = [ + { name = "h5py", specifier = ">=3" }, { name = "hypothesis", specifier = ">=6" }, { name = "pytest", specifier = ">=8" }, ]