diff --git a/tests/test_appraisal_resolution.py b/tests/test_appraisal_resolution.py new file mode 100644 index 0000000..a9aba0e --- /dev/null +++ b/tests/test_appraisal_resolution.py @@ -0,0 +1,202 @@ +"""The appraisal-resolution set proves itself: digests, referents, schema. + +No verifier resolves ``appraisal.policy_ref``, so there is no implementation to +run these against. What can be checked — and is checked here — is that the set +is internally what it claims to be: + + * every record is valid under the packaged schema on this branch's base, so + the set describes conformant records rather than malformed ones; + * every declared digest really is, or really is not, the SHA-256 of the bytes + the vector says the citation resolved to, matching its expected outcome; + * the unreachable vector really has no resolvable object; + * vector 06's algorithm really is outside the digest set the schema admits. + +A vector whose expected outcome and whose bytes disagree is worse than no +vector: it looks like coverage and argues for the wrong thing. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import jsonschema +import pytest + +VECTOR_DIR = Path(__file__).parent / "vectors" / "appraisal-resolution" +SCHEMA_DIGEST_PATTERN = re.compile(r"^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$") + + +def _vector_paths() -> list[Path]: + return sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json")) + + +def _load(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _sha256_of_file(rel: str) -> str: + return "sha256:" + hashlib.sha256((VECTOR_DIR / rel).read_bytes()).hexdigest() + + +VECTORS = _vector_paths() + + +@pytest.mark.level0 +def test_the_set_has_the_seven_vectors_it_documents() -> None: + names = [p.name for p in VECTORS] + assert len(names) == 7, names + for n, path in enumerate(VECTORS, start=1): + assert path.name.startswith(f"{n:02d}-"), ( + f"vectors must be contiguously numbered; got {path.name} at position {n}" + ) + + +@pytest.mark.level0 +@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem) +def test_the_record_is_valid_under_the_packaged_schema(path: Path, schema) -> None: + """Schema validity, so the set is about resolution and not about malformed input.""" + jsonschema.validate(_load(path)["record"], schema) + + +@pytest.mark.level0 +@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem) +def test_the_record_carries_a_bare_policy_ref_and_no_invented_field(path: Path) -> None: + """The candidate binding must never leak into the record. + + ``appraisal`` is ``additionalProperties: false`` in the packaged schema, so + a record carrying the candidate field would be schema-invalid — and + proposing it is a schema decision this set does not make. + """ + appraisal = _load(path)["record"]["appraisal"] + assert set(appraisal) <= {"status", "verifier", "policy_ref", "timestamp"}, ( + f"record appraisal carries an unexpected key: {sorted(appraisal)}" + ) + for key in appraisal: + assert "digest" not in key, f"record appraisal proposes a binding field: {key}" + + +@pytest.mark.level0 +@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem) +def test_the_declared_resolution_matches_the_bytes_on_disk(path: Path) -> None: + """Whatever the vector says resolution returned, the file must hash to it.""" + ctx = _load(path)["context"] + resolution = ctx["resolution"] + if resolution["outcome"] != "resolved": + assert resolution.get("file") in (None,), ( + "a resolution that did not resolve must not name a file" + ) + return + rel = resolution["file"] + assert (VECTOR_DIR / rel).is_file(), f"{rel} is missing from the set" + assert resolution["actual_sha256"] == _sha256_of_file(rel), ( + f"{rel} does not hash to the digest the vector records for it" + ) + + +@pytest.mark.level0 +@pytest.mark.negative +@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem) +def test_the_binding_agrees_with_the_expected_outcome(path: Path) -> None: + """The load-bearing self-proof. + + accept -> the declared binding equals the digest of what resolved + (or no binding is declared at all) + reject -> a binding and a resolution both exist, and they disagree + deferred-> the comparison could not be performed at all + """ + vector = _load(path) + ctx, outcome = vector["context"], vector["expected"]["outcome"] + binding = ctx.get("candidate_binding") + resolution = ctx["resolution"] + resolved = resolution["outcome"] == "resolved" + + if outcome == "pass": + if binding is None: + # 01: nothing was declared, so there is nothing that could contradict. + assert resolution["outcome"] == "not_attempted", ( + "a vector declaring no binding must not claim a resolution was " + "attempted; the point is that there was nothing to check" + ) + return + assert resolved, "an accepting vector with a binding must have resolved" + assert binding["value"] == resolution["actual_sha256"], ( + "accept requires the declared binding to equal what resolved" + ) + + elif outcome == "reject": + assert binding is not None, "a rejecting vector must declare a binding" + assert resolved, ( + "a rejecting vector must have resolved something; otherwise nothing " + "was contradicted and the case is unresolvable, not contradicted" + ) + assert SCHEMA_DIGEST_PATTERN.match(binding["value"]), ( + "a rejecting vector's binding must be well formed, so the rejection " + "is about the referent and not about a malformed digest" + ) + assert binding["value"] != resolution["actual_sha256"], ( + "reject requires the declared binding to differ from what resolved" + ) + + elif outcome == "deferred": + computable = ( + binding is not None + and SCHEMA_DIGEST_PATTERN.match(binding["value"]) is not None + ) + assert not (resolved and computable), ( + "a deferred vector must be one the verifier could not complete: " + "either the referent did not resolve, or the digest algorithm is " + "outside the set the schema admits" + ) + + else: # pragma: no cover - guarded by the completeness test + raise AssertionError(f"unknown expected outcome {outcome!r}") + + +@pytest.mark.level0 +@pytest.mark.negative +def test_05_really_has_no_resolvable_object() -> None: + vector = _load(VECTOR_DIR / "05-referent-unreachable.json") + assert vector["context"]["resolution"]["outcome"] == "unreachable" + cited = vector["context"]["cited_uri"] + tail = cited.rsplit("/", 1)[-1] + assert not (VECTOR_DIR / "policies" / tail).exists(), ( + "05 claims the referent is unreachable, but a sibling file matches its URI" + ) + + +@pytest.mark.level0 +@pytest.mark.negative +def test_06_names_an_algorithm_the_schema_does_not_admit() -> None: + vector = _load(VECTOR_DIR / "06-digest-algorithm-uncomputable.json") + value = vector["context"]["candidate_binding"]["value"] + assert not SCHEMA_DIGEST_PATTERN.match(value), ( + "06 claims the algorithm is uncomputable, but the digest matches the " + "pattern the schema admits" + ) + assert vector["context"]["resolution"]["outcome"] == "resolved", ( + "06 must reach its referent; that is what separates it from 05" + ) + + +@pytest.mark.level0 +def test_03_and_04_differ_in_kind_not_just_in_bytes() -> None: + """The pair that keeps the contradicted boundary off a single vector.""" + minimal = _load(VECTOR_DIR / "03-digest-mismatch-minimal-mutation.json") + wholesale = _load(VECTOR_DIR / "04-digest-mismatch-different-object.json") + + a = (VECTOR_DIR / minimal["context"]["resolution"]["file"]).read_bytes() + b = (VECTOR_DIR / "policies" / "appraisal-policy-v1.json").read_bytes() + assert len(a) == len(b), "03's mutation should not change the object's length" + assert sum(x != y for x, y in zip(a, b)) == 1, ( + "03 is the minimal-mutation vector; it must differ from the appraised " + "object in exactly one byte" + ) + + c = (VECTOR_DIR / wholesale["context"]["resolution"]["file"]).read_bytes() + assert len(c) != len(b), ( + "04 is the wholesale-substitution vector; a verifier comparing lengths " + "must be able to tell it from 03" + ) diff --git a/tests/test_appraisal_resolution_completeness.py b/tests/test_appraisal_resolution_completeness.py new file mode 100644 index 0000000..c358c57 --- /dev/null +++ b/tests/test_appraisal_resolution_completeness.py @@ -0,0 +1,194 @@ +"""Adequacy of the appraisal-resolution set, per the criteria on trace-spec#186. + +agentrust-io/trace-spec#186 (merged 2026-08-20) states what a conformance +vector set is claiming: *a verifier that does not implement these rules will +fail this set*. Three of its four criteria are checkable here and are checked; +the fourth is about repository-wide bookkeeping and is noted below. + +It merged into trace-spec, where it grades that repository's ``examples/``. +This repository has no adequacy harness, so nothing here is subject to it. +These criteria are a standard this set was built to by choice, and the tests +below are this set holding itself to them. + + 1. A set must fail BOTH unconditional implementations. + A set of all-rejections is passed by a verifier that rejects everything, + exactly as a set of all-acceptances is passed by one that accepts + everything. This set's three decided-reject vectors would, alone, be the + first failure. Vectors 01 and 02 exist to close it. + 2. Every boundary needs more than one vector. + One vector cannot separate a check that reads a prefix from one that + reads the whole object. + 3. Every set on disk is measured, or named with the test that measures it. + Repository-wide; trace-tests has no registry to add to, so it cannot be + asserted from inside one set. Recorded in the set's README instead. + 4. Shortfalls are recorded exactly. + See KNOWN_SHORTFALLS below. + +These tests grade the set, not a verifier. No verifier resolves +``appraisal.policy_ref`` today — that is the gap the set documents — so nothing +here claims an implementation was exercised. +""" + +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +import pytest + +VECTOR_DIR = Path(__file__).parent / "vectors" / "appraisal-resolution" + +DECIDED_OUTCOMES = {"pass", "reject"} +ALL_OUTCOMES = DECIDED_OUTCOMES | {"deferred"} + +# Criterion 4: shortfalls asserted to their exact extent, so they cannot widen +# quietly. Delete an entry when the shortfall is closed, not when it is excused. +KNOWN_SHORTFALLS = { + "no_verifier_exercised": ( + "No implementation resolves appraisal.policy_ref, so every expected " + "outcome is a claim about what a verifier should do, not a recording of " + "what one did. The set is a specification argument with runnable " + "internal consistency, not a conformance run." + ), + "unresolvable_outcome_unnamed": ( + "Vectors 05 and 06 assert only that the outcome is not affirming. The " + "value a verifier should record is open on agentrust-io/trace-spec#190 " + "and is deliberately not proposed here." + ), +} + + +def _vectors() -> list[dict]: + out = [] + for path in sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json")): + out.append(json.loads(path.read_text(encoding="utf-8"))) + return out + + +def _outcome(vector: dict) -> str: + return vector["expected"]["outcome"] + + +@pytest.mark.level0 +def test_the_set_is_not_empty() -> None: + assert len(_vectors()) == 7, "the set is seven vectors, 01 through 07" + + +@pytest.mark.level0 +def test_criterion_1_the_set_fails_an_accept_everything_verifier() -> None: + """At least one vector a conformant verifier must reject.""" + rejects = [v for v in _vectors() if _outcome(v) == "reject"] + assert rejects, ( + "every vector expects acceptance, so a verifier that accepts " + "unconditionally passes the set" + ) + + +@pytest.mark.level0 +def test_criterion_1_the_set_fails_a_reject_everything_verifier() -> None: + """At least one vector a conformant verifier must accept. + + This is the criterion the set would otherwise fail. Its subject is a family + of resolution failures, so every vector written from the motivating problem + alone is a reject or a deferral. + """ + accepts = [v for v in _vectors() if _outcome(v) == "pass"] + assert accepts, ( + "no vector expects acceptance, so a verifier that rejects " + "unconditionally passes the set" + ) + assert len(accepts) >= 2, ( + "one must-accept vector cannot separate a verifier that accepts only " + "records declaring no binding from one that also checks a matching " + "binding; 01 and 02 are that pair" + ) + + +@pytest.mark.level0 +def test_criterion_2_every_boundary_carries_at_least_two_vectors() -> None: + counts = Counter(v["boundary"] for v in _vectors()) + thin = {b: n for b, n in counts.items() if n < 2} + assert not thin, f"boundaries carried by a single vector: {thin}" + assert set(counts) == {"accept", "contradicted", "unresolvable"}, ( + f"unexpected boundary set: {sorted(counts)}" + ) + + +@pytest.mark.level0 +def test_no_two_vectors_share_a_defect() -> None: + defects = [v["defect"] for v in _vectors()] + dupes = [d for d, n in Counter(defects).items() if n > 1 and d != "none"] + assert not dupes, f"defect exercised by more than one vector: {dupes}" + + +@pytest.mark.level0 +def test_every_expected_block_is_well_formed() -> None: + for v in _vectors(): + name = v["name"] + exp = v["expected"] + assert exp["outcome"] in ALL_OUTCOMES, f"{name}: bad outcome {exp['outcome']!r}" + assert exp.get("reason"), f"{name}: expected block carries no reason" + if exp["outcome"] == "deferred": + assert exp.get("deferred_pending") == "agentrust-io/trace-spec#190", ( + f"{name}: a deferred vector must name the issue it defers to" + ) + assert exp.get("must_not") == "affirming", ( + f"{name}: a deferred vector must still assert what it may not be" + ) + else: + assert "deferred_pending" not in exp, ( + f"{name}: a decided vector must not carry a deferral pointer" + ) + + +@pytest.mark.level0 +def test_no_vector_proposes_an_appraisal_status_value() -> None: + """The set must not coin the vocabulary trace-spec#190 exists to decide. + + ``deferred`` is fixture bookkeeping. It must never appear as an + ``appraisal.status``, and no vector may assert a status for the + unresolvable case. + """ + schema_enum = {"affirming", "warning", "contraindicated", "none"} + for v in _vectors(): + status = v["record"]["appraisal"]["status"] + assert status in schema_enum, f"{v['name']}: invented status {status!r}" + exp = v["expected"] + assert exp["outcome"] not in schema_enum, ( + f"{v['name']}: expected.outcome reuses an appraisal.status value, " + "which reads as proposing that value for this case" + ) + if exp["outcome"] == "deferred": + assert "status" not in exp, ( + f"{v['name']}: a deferred vector must not name a status to record" + ) + + +@pytest.mark.level0 +def test_candidate_binding_is_marked_candidate_everywhere_it_appears() -> None: + for v in _vectors(): + binding = v["context"].get("candidate_binding") + if binding is None: + continue + assert binding["field"].startswith("CANDIDATE:"), ( + f"{v['name']}: the binding field must be marked CANDIDATE, so it is " + "never read as a proposed schema field" + ) + + +@pytest.mark.level0 +def test_known_shortfalls_are_recorded_not_silent() -> None: + """Criterion 4: the gaps are asserted to their exact extent.""" + assert set(KNOWN_SHORTFALLS) == { + "no_verifier_exercised", + "unresolvable_outcome_unnamed", + }, ( + "the recorded shortfalls changed; update the README in the same commit " + "so the set never claims more coverage than it has" + ) + deferred = [v for v in _vectors() if _outcome(v) == "deferred"] + assert len(deferred) == 2, ( + "unresolvable_outcome_unnamed is recorded as covering exactly two " + f"vectors; found {len(deferred)}" + ) diff --git a/tests/test_appraisal_resolution_reproduces.py b/tests/test_appraisal_resolution_reproduces.py new file mode 100644 index 0000000..af62429 --- /dev/null +++ b/tests/test_appraisal_resolution_reproduces.py @@ -0,0 +1,106 @@ +"""The appraisal-resolution generator must reproduce its committed vectors. + +A committed vector nobody can regenerate is a number that cannot be checked. If +the generator and the files drift, the files win by default and the drift is +invisible, which is the failure agentrust-io/trace-spec#171 was written to +catch for that repository's ``examples/``. + +trace-tests has no equivalent guard, and trace-tests PR #66 states the reason a +cross-repository one is not the answer: "a guard that needs another repository +checked out is a guard that gets skipped." This guard is therefore +self-contained — it imports the generator that sits beside the vectors and +regenerates into a temporary directory, comparing bytes. + +Regenerating into ``tmp_path`` rather than in place is the load-bearing part. +Running the generator over its own directory and then comparing those files to +themselves agrees no matter what the generator does. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +VECTOR_DIR = Path(__file__).parent / "vectors" / "appraisal-resolution" +GENERATOR = VECTOR_DIR / "gen_appraisal_resolution.py" + + +def _load_generator(): + spec = importlib.util.spec_from_file_location("gen_appraisal_resolution", GENERATOR) + assert spec and spec.loader, "generator module could not be loaded" + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _committed_files() -> list[Path]: + return sorted(p for p in VECTOR_DIR.rglob("*.json") if p.is_file()) + + +@pytest.mark.level0 +def test_the_generator_is_present_beside_the_vectors() -> None: + """Self-containment: the guard must not need another repository.""" + assert GENERATOR.is_file(), ( + "gen_appraisal_resolution.py must sit in the vector directory, so the " + "set can be regenerated by anyone holding only that directory" + ) + + +@pytest.mark.level0 +def test_the_generator_reproduces_every_committed_file(tmp_path: Path) -> None: + module = _load_generator() + module.main(tmp_path) + + committed = _committed_files() + assert committed, "no vectors found; the set is empty" + + mismatched: list[str] = [] + missing: list[str] = [] + for path in committed: + rel = path.relative_to(VECTOR_DIR) + produced = tmp_path / rel + if not produced.is_file(): + missing.append(str(rel)) + continue + if produced.read_bytes() != path.read_bytes(): + mismatched.append(str(rel)) + + assert not missing, f"generator did not produce: {missing}" + assert not mismatched, ( + f"generator output differs from the committed bytes: {mismatched}. " + "Re-run tests/vectors/appraisal-resolution/gen_appraisal_resolution.py " + "and commit the result, or fix the generator — but do not edit a vector " + "by hand and leave the generator behind." + ) + + +@pytest.mark.level0 +def test_the_generator_produces_nothing_the_set_does_not_carry(tmp_path: Path) -> None: + """Both directions: a file the generator emits but nobody committed is drift too.""" + module = _load_generator() + module.main(tmp_path) + + produced = {p.relative_to(tmp_path) for p in tmp_path.rglob("*.json") if p.is_file()} + committed = {p.relative_to(VECTOR_DIR) for p in _committed_files()} + assert produced == committed, ( + f"only generated: {sorted(str(p) for p in produced - committed)}; " + f"only committed: {sorted(str(p) for p in committed - produced)}" + ) + + +@pytest.mark.level0 +def test_the_committed_bytes_use_lf_and_ascii_only() -> None: + """The two properties the digests depend on, asserted rather than assumed. + + CRLF translation changes every policy digest. ``.gitattributes`` in the + vector directory pins ``eol=lf``; this fails loudly if that protection is + ever removed. ASCII-only is separate: ``conftest.load_vector`` reads with + ``Path.read_text()`` and no explicit encoding, so a non-ASCII byte would be + decoded under the platform's locale. + """ + for path in _committed_files(): + data = path.read_bytes() + assert b"\r\n" not in data, f"{path.name} contains CRLF; digests will not match" + assert all(b < 128 for b in data), f"{path.name} contains a non-ASCII byte" diff --git a/tests/vectors/appraisal-resolution/.gitattributes b/tests/vectors/appraisal-resolution/.gitattributes new file mode 100644 index 0000000..69a8d4a --- /dev/null +++ b/tests/vectors/appraisal-resolution/.gitattributes @@ -0,0 +1,13 @@ +# The digests carried in these vectors are SHA-256 over the exact bytes of the +# sibling files under policies/. Line-ending translation therefore changes the +# answer: with core.autocrlf=true (the Git for Windows default) an unmarked +# checkout rewrites LF to CRLF, every policy digest stops matching, and both the +# byte-reproduction guard and the self-proof tests fail on a clean clone. +# +# Verified, not assumed: without this file, deleting +# policies/appraisal-policy-v1.json and restoring it with `git checkout --` +# changed its SHA-256 from d8764863... to 7e68506c... on a Windows worktree. +# +# `text eol=lf` keeps these diffable as text in review while pinning the working +# tree to LF on every platform. +* text eol=lf diff --git a/tests/vectors/appraisal-resolution/01-no-binding-declared.json b/tests/vectors/appraisal-resolution/01-no-binding-declared.json new file mode 100644 index 0000000..02dc208 --- /dev/null +++ b/tests/vectors/appraisal-resolution/01-no-binding-declared.json @@ -0,0 +1,57 @@ +{ + "name": "no-binding-declared", + "description": "The record cites an appraisal policy by bare URI and declares no binding to what that URI held. This is every conformant record today, so it must keep verifying: a set that rejected it would be proposing a breaking change rather than describing a gap.", + "boundary": "accept", + "defect": "none - backward-compatibility control", + "spec": "Mirrors the merged treatment of an undeclared depth in agentrust-io/trace-spec#173, where a record that never declared is read as surface rather than as a failure.", + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1748000000, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5" + }, + "runtime": { + "platform": "intel-tdx", + "measurement": "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8" + }, + "policy": { + "bundle_hash": "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example.org", + "policy_ref": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "timestamp": 1748000042 + }, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001" + } + } + }, + "context": { + "cited_uri": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "resolution": { + "outcome": "not_attempted", + "note": "No binding is declared, so there is nothing to check." + }, + "candidate_binding": null + }, + "expected": { + "outcome": "pass", + "reason": "No binding declared; nothing to contradict." + } +} diff --git a/tests/vectors/appraisal-resolution/02-resolved-and-matches.json b/tests/vectors/appraisal-resolution/02-resolved-and-matches.json new file mode 100644 index 0000000..eb9b376 --- /dev/null +++ b/tests/vectors/appraisal-resolution/02-resolved-and-matches.json @@ -0,0 +1,61 @@ +{ + "name": "resolved-and-matches", + "description": "The cited URI resolves and its bytes hash to exactly the declared candidate binding. The second must-accept vector, and the only one in which a resolution actually succeeds.", + "boundary": "accept", + "defect": "none - positive control", + "spec": "agentrust-io/trace-spec docs/verification.md - a verifier records what it actually resolved, and evidence that resolves and contradicts fails the appraisal. Applied here to appraisal.policy_ref, which carries no digest.", + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1748000000, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5" + }, + "runtime": { + "platform": "intel-tdx", + "measurement": "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8" + }, + "policy": { + "bundle_hash": "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example.org", + "policy_ref": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "timestamp": 1748000042 + }, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001" + } + } + }, + "context": { + "cited_uri": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "resolution": { + "outcome": "resolved", + "file": "policies/appraisal-policy-v1.json", + "actual_sha256": "sha256:d8764863e75e702ff64e53951eb3d005a84858979598f7f0bda11fc901416adc" + }, + "candidate_binding": { + "field": "CANDIDATE:appraisal.policy_digest", + "value": "sha256:d8764863e75e702ff64e53951eb3d005a84858979598f7f0bda11fc901416adc" + } + }, + "expected": { + "outcome": "pass", + "reason": "Declared binding equals the digest of what resolved." + } +} diff --git a/tests/vectors/appraisal-resolution/03-digest-mismatch-minimal-mutation.json b/tests/vectors/appraisal-resolution/03-digest-mismatch-minimal-mutation.json new file mode 100644 index 0000000..ef417e2 --- /dev/null +++ b/tests/vectors/appraisal-resolution/03-digest-mismatch-minimal-mutation.json @@ -0,0 +1,62 @@ +{ + "name": "digest-mismatch-minimal-mutation", + "description": "The cited URI now serves a document one character from the one appraised: the SLSA floor moved from 2 to 3. The record still declares the original digest. A verifier that compares only the URI sees no change; a verifier that compares bytes sees the substitution that flips this record's verdict.", + "boundary": "contradicted", + "defect": "cited object mutated minimally after appraisal", + "spec": "agentrust-io/trace-spec docs/verification.md - a verifier records what it actually resolved, and evidence that resolves and contradicts fails the appraisal. Applied here to appraisal.policy_ref, which carries no digest.", + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1748000000, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5" + }, + "runtime": { + "platform": "intel-tdx", + "measurement": "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8" + }, + "policy": { + "bundle_hash": "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example.org", + "policy_ref": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "timestamp": 1748000042 + }, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001" + } + } + }, + "context": { + "cited_uri": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "resolution": { + "outcome": "resolved", + "file": "policies/appraisal-policy-v1-rev2.json", + "actual_sha256": "sha256:1248932d38677845463158e5b93b7d1e3f78b1fffdef6d23e25c1d2926d5e3d5" + }, + "candidate_binding": { + "field": "CANDIDATE:appraisal.policy_digest", + "value": "sha256:d8764863e75e702ff64e53951eb3d005a84858979598f7f0bda11fc901416adc" + }, + "note": "policies/appraisal-policy-v1.json and policies/appraisal-policy-v1-rev2.json differ in one character." + }, + "expected": { + "outcome": "reject", + "reason": "Resolved bytes contradict the declared binding." + } +} diff --git a/tests/vectors/appraisal-resolution/04-digest-mismatch-different-object.json b/tests/vectors/appraisal-resolution/04-digest-mismatch-different-object.json new file mode 100644 index 0000000..a86fe63 --- /dev/null +++ b/tests/vectors/appraisal-resolution/04-digest-mismatch-different-object.json @@ -0,0 +1,61 @@ +{ + "name": "digest-mismatch-different-object", + "description": "The cited URI resolves to an unrelated policy document. Paired with 03 so the boundary is not carried by a single vector: a verifier that only samples a prefix of the document, or compares lengths, passes one of these two and fails the other.", + "boundary": "contradicted", + "defect": "cited object wholly replaced after appraisal", + "spec": "agentrust-io/trace-spec docs/verification.md - a verifier records what it actually resolved, and evidence that resolves and contradicts fails the appraisal. Applied here to appraisal.policy_ref, which carries no digest.", + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1748000000, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5" + }, + "runtime": { + "platform": "intel-tdx", + "measurement": "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8" + }, + "policy": { + "bundle_hash": "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example.org", + "policy_ref": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "timestamp": 1748000042 + }, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001" + } + } + }, + "context": { + "cited_uri": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "resolution": { + "outcome": "resolved", + "file": "policies/unrelated-policy.json", + "actual_sha256": "sha256:23905737177574be26e1212264b88ffe81e06e99aab1fe6115ebbb2e9205109c" + }, + "candidate_binding": { + "field": "CANDIDATE:appraisal.policy_digest", + "value": "sha256:d8764863e75e702ff64e53951eb3d005a84858979598f7f0bda11fc901416adc" + } + }, + "expected": { + "outcome": "reject", + "reason": "Resolved bytes contradict the declared binding." + } +} diff --git a/tests/vectors/appraisal-resolution/05-referent-unreachable.json b/tests/vectors/appraisal-resolution/05-referent-unreachable.json new file mode 100644 index 0000000..cbf5e9f --- /dev/null +++ b/tests/vectors/appraisal-resolution/05-referent-unreachable.json @@ -0,0 +1,63 @@ +{ + "name": "referent-unreachable", + "description": "The cited URI does not resolve; no object under policies/ corresponds to it. Nothing was contradicted, because nothing was read. This vector asserts only that the outcome is not affirming.", + "boundary": "unresolvable", + "defect": "referent unreachable at verification time", + "spec": "agentrust-io/trace-spec docs/verification.md - evidence that does not resolve downgrades honestly rather than failing. Which value records that for appraisal.policy_ref is open: agentrust-io/trace-spec#190. This vector asserts only what the outcome may NOT be.", + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1748000000, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5" + }, + "runtime": { + "platform": "intel-tdx", + "measurement": "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8" + }, + "policy": { + "bundle_hash": "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example.org", + "policy_ref": "https://policy.example.org/appraisal/appraisal-policy-withdrawn.json", + "timestamp": 1748000042 + }, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001" + } + } + }, + "context": { + "cited_uri": "https://policy.example.org/appraisal/appraisal-policy-withdrawn.json", + "resolution": { + "outcome": "unreachable", + "file": null, + "note": "No sibling file corresponds to this URI, by construction." + }, + "candidate_binding": { + "field": "CANDIDATE:appraisal.policy_digest", + "value": "sha256:d8764863e75e702ff64e53951eb3d005a84858979598f7f0bda11fc901416adc" + } + }, + "expected": { + "outcome": "deferred", + "deferred_pending": "agentrust-io/trace-spec#190", + "must_not": "affirming", + "reason": "Reporting a check that was never performed as affirming is the failure this set exists to name. Which value is reported instead is not decided here." + } +} diff --git a/tests/vectors/appraisal-resolution/06-digest-algorithm-uncomputable.json b/tests/vectors/appraisal-resolution/06-digest-algorithm-uncomputable.json new file mode 100644 index 0000000..27a541f --- /dev/null +++ b/tests/vectors/appraisal-resolution/06-digest-algorithm-uncomputable.json @@ -0,0 +1,64 @@ +{ + "name": "digest-algorithm-uncomputable", + "description": "The referent resolves, but the declared binding names a digest algorithm outside the set this profile's schema admits (sha256 and sha384). The verifier cannot compute the comparison, so it has not checked - which is distinct from having checked and disagreed. Paired with 05: one cannot reach the object, the other reaches it and cannot compute over it.", + "boundary": "unresolvable", + "defect": "digest algorithm the verifier cannot compute", + "spec": "agentrust-io/trace-spec docs/verification.md - evidence that does not resolve downgrades honestly rather than failing. Which value records that for appraisal.policy_ref is open: agentrust-io/trace-spec#190. This vector asserts only what the outcome may NOT be. Deliberately rhymes with the unverifiable / digest_algorithm_unsupported classification PROPOSED in agentrust-io/trace-spec#184 - an explicitly non-normative draft, a surface facing this question rather than an established answer to it - without adopting its vocabulary.", + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1748000000, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5" + }, + "runtime": { + "platform": "intel-tdx", + "measurement": "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8" + }, + "policy": { + "bundle_hash": "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example.org", + "policy_ref": "https://policy.example.org/appraisal/appraisal-policy-v2.json", + "timestamp": 1748000042 + }, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001" + } + } + }, + "context": { + "cited_uri": "https://policy.example.org/appraisal/appraisal-policy-v2.json", + "resolution": { + "outcome": "resolved", + "file": "policies/appraisal-policy-v2.json", + "actual_sha256": "sha256:9dd48a5dc2efc67c3873a38c95aae17bc49345b48d09e6629f177ac0a5ff79cc" + }, + "candidate_binding": { + "field": "CANDIDATE:appraisal.policy_digest", + "value": "sha3-512:49c36096f3c7cf58d58324f53502f427a72b0684776fe6006f8a8b13513a9c40d626893b68102cec8c8c78d871814067bab429a7d09e5ff4ef14147eb35ef9a5", + "note": "This is the correct SHA3-512 of the referent. It is outside the schema's digest pattern ^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$, so no conformant verifier is required to compute it - which is the only reason this vector is undecided." + } + }, + "expected": { + "outcome": "deferred", + "deferred_pending": "agentrust-io/trace-spec#190", + "must_not": "affirming", + "reason": "The comparison was not performed, so the result is not a contradiction. Which value records that is not decided here." + } +} diff --git a/tests/vectors/appraisal-resolution/07-digest-bound-to-other-referent.json b/tests/vectors/appraisal-resolution/07-digest-bound-to-other-referent.json new file mode 100644 index 0000000..5024ecc --- /dev/null +++ b/tests/vectors/appraisal-resolution/07-digest-bound-to-other-referent.json @@ -0,0 +1,62 @@ +{ + "name": "digest-bound-to-other-referent", + "description": "The declared binding is well formed and is the true digest of a real object in this set - version 2 - while policy_ref cites version 1. Both halves are individually valid and the pair is not. A verifier that checks the digest is well formed, or that it matches something it holds, passes this; only one that checks the digest against what this URI resolved to rejects it.", + "boundary": "contradicted", + "defect": "binding well formed but bound to a different referent", + "spec": "agentrust-io/trace-spec docs/verification.md - a verifier records what it actually resolved, and evidence that resolves and contradicts fails the appraisal. Applied here to appraisal.policy_ref, which carries no digest.", + "record": { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": 1748000000, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": { + "provider": "anthropic", + "model_id": "claude-sonnet-4-5" + }, + "runtime": { + "platform": "intel-tdx", + "measurement": "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8" + }, + "policy": { + "bundle_hash": "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce" + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8" + }, + "appraisal": { + "status": "affirming", + "verifier": "https://verifier.example.org", + "policy_ref": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "timestamp": 1748000042 + }, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001" + } + } + }, + "context": { + "cited_uri": "https://policy.example.org/appraisal/appraisal-policy-v1.json", + "resolution": { + "outcome": "resolved", + "file": "policies/appraisal-policy-v1.json", + "actual_sha256": "sha256:d8764863e75e702ff64e53951eb3d005a84858979598f7f0bda11fc901416adc" + }, + "candidate_binding": { + "field": "CANDIDATE:appraisal.policy_digest", + "value": "sha256:9dd48a5dc2efc67c3873a38c95aae17bc49345b48d09e6629f177ac0a5ff79cc", + "note": "This is the digest of policies/appraisal-policy-v2.json, which is not what cited_uri resolves to." + } + }, + "expected": { + "outcome": "reject", + "reason": "The binding does not describe the object the record cites, even though it describes some object." + } +} diff --git a/tests/vectors/appraisal-resolution/README.md b/tests/vectors/appraisal-resolution/README.md new file mode 100644 index 0000000..c353ee9 --- /dev/null +++ b/tests/vectors/appraisal-resolution/README.md @@ -0,0 +1,173 @@ +# Appraisal-resolution vectors + +Candidate conformance vectors for one question: **when a record cites an +appraisal policy by URI, what can a second verifier confirm about what that URI +held?** + +Today, nothing. `appraisal.policy_ref` is a bare URI. The record carries a +digest for the enforcement policy bundle (`policy.bundle_hash`) and none for the +appraisal policy that produced the verdict, so two verifiers resolving the same +`policy_ref` at different times can retrieve different documents and both report +`affirming` honestly. + +## Why this set exists + +`agentrust-io/trace-tests#63` proposed a conformance module for the `appraisal` +claim, noting that `policy_ref` can be checked for well-formedness only, "since +the record carries no digest to compare a resolved policy against." Building +those cases ran into the boundary itself, which was stated on +`agentrust-io/trace-spec#66` on 2026-08-18: + +> `appraisal.policy_ref` can be checked for well-formedness, but not +> reproduced: the record carries nothing stating what the referent was, so a +> conformance module can confirm the URI parses and nothing more. Two verifiers +> resolving the same `policy_ref` at different times can retrieve different +> documents and both honestly report verified — the federation gap §1 names, +> one hop from the record. + +Drafting candidate fixtures for it was accepted on that thread the same day, +with the shape asked for: candidates in `trace-tests`, one defect per vector, +expected outcomes committed beside the vectors, and the schema and +`verification.md` text left maintainer-authored. This set is that. + +## What it does not do + +**It does not propose a schema change.** `candidate_binding` is a *candidate +shape*, marked `CANDIDATE:` in every vector, and it lives in the vector's +`context` — never inside `record`. The packaged schema sets +`additionalProperties: false` on `appraisal`, so a record carrying an extra +field would be schema-invalid; whether a binding field lands, and what it is +called, is an editorial decision. + +**It does not name an outcome value for the unresolvable case.** Vectors 05 and +06 are marked `deferred`, pointing at `agentrust-io/trace-spec#190`, which +tracks the open cross-surface question: *when a record cites something a +verifier cannot resolve, what does the verifier record, and where.* + +Four surfaces face that question. Only one has answered it: +`build_provenance`, with a recorded field plus a floor +(`agentrust-io/trace-spec#173`, **merged**). Revocation states the prohibition +in prose with no field to record the outcome in +(`agentrust-io/trace-spec#187`, **merged**). Delegation links **propose** +`unverifiable` on a separate axis (`agentrust-io/trace-spec#184`, **an +explicitly non-normative draft** — a proposal facing the question, not an +established answer; its author has said so on +`agentrust-io/trace-spec#190`). And this one. + +The divergence is a schema fact rather than four differing opinions: +`appraisal` is `additionalProperties: false`, and `#173` landed the only +claimed/verified pair that exists, so the other surfaces had no field to use. +Coining a fifth answer here is exactly what `agentrust-io/trace-spec#190` +exists to prevent — and a new field or a fifth `appraisal.status` value is +normative under `CONTRIBUTING.md`, which routes it through a Spec change +proposal with a sponsoring organization. It is not something a vector set +decides. + +> **`deferred` is fixture bookkeeping, not a proposed appraisal vocabulary +> value.** It is a property of a *vector's expected block*, recording that the +> outcome is not yet decided upstream. It is **not** a candidate value for +> `appraisal.status`, which is closed at `affirming`, `warning`, +> `contraindicated`, `none`. A deferred vector asserts only what the outcome may +> **not** be — `must_not: "affirming"` — because reporting a check that was +> never performed as affirming is the one thing every reading of the merged text +> already rules out. `tests/test_appraisal_resolution_completeness.py` fails if +> any vector reuses a status value as an outcome. + +## The vectors + +Every record is identical except for `appraisal.policy_ref`, so the defect under +test is the only thing that varies. All records are unsigned and ASCII-only. + +| # | Vector | Boundary | Expected | +|---|---|---|---| +| 01 | `no-binding-declared` | accept | `pass` | +| 02 | `resolved-and-matches` | accept | `pass` | +| 03 | `digest-mismatch-minimal-mutation` | contradicted | `reject` | +| 04 | `digest-mismatch-different-object` | contradicted | `reject` | +| 05 | `referent-unreachable` | unresolvable | `deferred` | +| 06 | `digest-algorithm-uncomputable` | unresolvable | `deferred` | +| 07 | `digest-bound-to-other-referent` | contradicted | `reject` | + +**01 and 02 are the must-accept pair, and they are why this set is not +one-directional.** `agentrust-io/trace-spec#186` (merged 2026-08-20) states +the criterion: a set must fail *both* unconditional implementations. A set +written from the motivating problem alone would be all rejections and +deferrals, and a verifier that rejects everything would pass it. 01 is the +backward-compatibility control — every conformant record today declares no +binding, and must keep verifying, or this set would be proposing a breaking +change rather than describing a gap. 02 is the only vector in which a +resolution succeeds. + +**03 and 04 keep the contradicted boundary off a single vector.** 03 differs +from the appraised object in exactly one byte — the SLSA floor moves 2 → 3, +which flips this record's verdict. 04 substitutes an unrelated document of a +different length. A verifier comparing lengths, or sampling a prefix, passes one +and fails the other. + +**05 and 06 are both unresolvable, by different mechanisms.** 05 cannot reach +the object; 06 reaches it and cannot compute over it, because the declared +algorithm (`sha3-512`) is outside the set the schema admits +(`^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$`). The distinction deliberately +rhymes with the `digest_algorithm_unsupported` classification **proposed** in +`agentrust-io/trace-spec#184` — a non-normative draft — without adopting that +vocabulary. + +**07 is the one a well-formedness check passes.** The binding is a valid +`sha256:` digest and is the true digest of a real object in this set — version 2 +— while `policy_ref` cites version 1. Both halves are individually valid; the +pair is not. + +## Reproducing it + +``` +python tests/vectors/appraisal-resolution/gen_appraisal_resolution.py +``` + +Deterministic: no keys, no clock, no randomness, no network. The digests are +SHA-256 over the exact bytes of the sibling files under `policies/`, so anyone +holding only this directory can recompute every number in the set. + +`tests/test_appraisal_resolution_reproduces.py` holds the generator to +byte-reproduction by regenerating into a temporary directory and comparing — +not in place, which would compare the files to themselves and agree regardless. + +The guard is **self-contained**. `agentrust-io/trace-spec#171` provides the +equivalent for that repository's `examples/`, and `trace-tests` has no such +registry; `agentrust-io/trace-tests#66` gives the reason not to reach across for +one — *"a guard that needs another repository checked out is a guard that gets +skipped."* + +`.gitattributes` in this directory pins `eol=lf`. This is load-bearing rather +than tidy: with `core.autocrlf=true`, a checkout rewrites LF to CRLF, every +policy digest stops matching, and the set fails on a clean clone. + +## What this set does not establish + +- **No verifier was exercised.** Nothing in `trace-tests` resolves + `appraisal.policy_ref` today — that is the gap — so every expected outcome is + a claim about what a verifier should do, not a recording of what one did. The + tests grade the set's internal consistency: that each declared digest really + is, or really is not, the digest of the bytes the vector says resolution + returned. +- **The unresolvable outcome is unnamed**, by choice, pending + `agentrust-io/trace-spec#190`. + +Both are recorded as exact shortfalls in +`tests/test_appraisal_resolution_completeness.py::KNOWN_SHORTFALLS`, which fails +if the list changes without this file changing with it. + +## Related + +- `agentrust-io/trace-tests#63` — the module proposal these cases were built for +- `agentrust-io/trace-spec#66` — where the gap was raised and the fixtures accepted +- `agentrust-io/trace-spec#190` — the deferred cross-surface question +- `agentrust-io/trace-spec#173` — merged: recorded field plus a policy floor +- `agentrust-io/trace-spec#184` — open, **draft, explicitly non-normative**: + *proposes* `unverifiable` on the delegation surface +- `agentrust-io/trace-spec#186` — merged: the adequacy criteria this set was + built to. It grades trace-spec's `examples/`; this repository has no adequacy + harness, so the standard is one this set chose, not one imposed on it +- `agentrust-io/trace-tests#66` — merged: `tr_sig` canonicalizes with RFC 8785; + source of the self-containment principle quoted above +- `agentrust-io/trace-tests#68` — merged: packaged schema resynced to the + normative v0.2 copy these records validate against diff --git a/tests/vectors/appraisal-resolution/gen_appraisal_resolution.py b/tests/vectors/appraisal-resolution/gen_appraisal_resolution.py new file mode 100644 index 0000000..1a28246 --- /dev/null +++ b/tests/vectors/appraisal-resolution/gen_appraisal_resolution.py @@ -0,0 +1,452 @@ +"""Regenerate the appraisal-resolution vector set, byte for byte. + +Deterministic by construction: no keys, no clock, no randomness, no network. +Running this on any machine with the same CPython minor version reproduces +every file in this directory exactly, which is what +``tests/test_appraisal_resolution_reproduces.py`` asserts. + + python tests/vectors/appraisal-resolution/gen_appraisal_resolution.py + +The digests in the vectors are SHA-256 over the exact bytes of the sibling +files under ``policies/``. Anyone holding only this directory can recompute +them; nothing here depends on another repository being checked out. + +WHAT THIS SET IS FOR + ``appraisal.policy_ref`` is a bare URI. A record says which appraisal + policy produced its verdict, but carries nothing stating what that URI + resolved to at appraisal time, so a second verifier cannot confirm it + retrieved the same document. See the set's README.md. + +WHAT IT DELIBERATELY DOES NOT DO + It does not name an outcome value for the unresolvable case. That is the + open cross-surface question tracked by agentrust-io/trace-spec#190, and + coining a value here is precisely what that issue exists to prevent. + Vectors 05 and 06 carry the fixture-bookkeeping state ``deferred``. + + ``candidate_binding`` is a CANDIDATE shape, not a proposed schema change. + It lives in the vector's ``context``, never inside ``record``: the + packaged schema sets ``additionalProperties: false`` on ``appraisal``, so + a record carrying an extra field would be schema-invalid, and proposing + one is a schema decision that belongs to the editorial process. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +HERE = Path(__file__).parent +POLICIES = HERE / "policies" + +# --- house serialization, fixed so bytes are stable across platforms ------- +INDENT = 2 + + +def write_json(path: Path, obj: object) -> bytes: + """Write *obj* as UTF-8 JSON with LF endings; return the exact bytes.""" + text = json.dumps(obj, indent=INDENT, ensure_ascii=True) + "\n" + data = text.encode("utf-8") + path.write_bytes(data) + return data + + +def sha256_of(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def sha3_512_of(data: bytes) -> str: + """A *correct* digest in an algorithm the profile's schema does not admit. + + Vector 06 turns on the verifier being unable to compute the comparison, not + on the digest being wrong. A placeholder value would confound the two: a + reader could not tell whether the vector is deferred because the algorithm + is unsupported or because the digest is obviously bogus. This is the real + SHA3-512 of the referent, so the algorithm is the only variable. + """ + return "sha3-512:" + hashlib.sha3_512(data).hexdigest() + + +# --- the cited objects ----------------------------------------------------- +# Small, ASCII-only, and shaped like an appraisal policy rather than a +# placeholder, so a reader can see why swapping one for another matters. + +POLICY_V1 = { + "policy_id": "appraisal/baseline", + "version": "1.0.0", + "rules": [ + {"claim": "runtime.platform", + "must_be_one_of": ["intel-tdx", "amd-sev-snp", "tpm2"]}, + {"claim": "build_provenance.slsa_level", "minimum": 2}, + ], +} + +# One character apart from V1: the SLSA floor moves 2 -> 3. A verifier +# applying this instead of V1 reaches a different verdict on the same record, +# which is why a minimal mutation is the honest test rather than a cosmetic one. +POLICY_V1_REV2 = { + "policy_id": "appraisal/baseline", + "version": "1.0.0", + "rules": [ + {"claim": "runtime.platform", + "must_be_one_of": ["intel-tdx", "amd-sev-snp", "tpm2"]}, + {"claim": "build_provenance.slsa_level", "minimum": 3}, + ], +} + +# A legitimate later version, published at its own URI. +POLICY_V2 = { + "policy_id": "appraisal/baseline", + "version": "2.0.0", + "rules": [ + {"claim": "runtime.platform", + "must_be_one_of": ["intel-tdx", "amd-sev-snp"]}, + {"claim": "build_provenance.slsa_level", "minimum": 3}, + {"claim": "transparency", "must_be_present": True}, + ], +} + +# A different policy entirely, not a version of the baseline. +POLICY_UNRELATED = { + "policy_id": "retention/pii-90d", + "version": "1.4.2", + "rules": [ + {"claim": "data_class", "must_be_one_of": ["public", "internal"]}, + ], +} + +POLICY_FILES = { + "appraisal-policy-v1.json": POLICY_V1, + "appraisal-policy-v1-rev2.json": POLICY_V1_REV2, + "appraisal-policy-v2.json": POLICY_V2, + "unrelated-policy.json": POLICY_UNRELATED, +} + +BASE_URI = "https://policy.example.org/appraisal/" + +# --- the record ------------------------------------------------------------ +# Every vector's record is identical except for appraisal.policy_ref, so the +# defect under test is the only thing that varies. Modelled on the repository's +# tests/vectors/valid_level0.json: unsigned, ASCII-only, fixed iat. + +RECORD_IAT = 1748000000 +APPRAISAL_TIMESTAMP = 1748000042 + + +def record_with(policy_ref: str | None) -> dict[str, object]: + appraisal: dict[str, object] = { + "status": "affirming", + "verifier": "https://verifier.example.org", + } + if policy_ref is not None: + appraisal["policy_ref"] = policy_ref + appraisal["timestamp"] = APPRAISAL_TIMESTAMP + return { + "eat_profile": "tag:agentrust-io.com,2026:trace-v0.2", + "iat": RECORD_IAT, + "subject": "spiffe://example.org/agent/credit-risk/01926b4c-1234-7abc-9def-000000000001", + "model": {"provider": "anthropic", "model_id": "claude-sonnet-4-5"}, + "runtime": { + "platform": "intel-tdx", + "measurement": + "sha256:a3f8d2b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8", + }, + "policy": { + "bundle_hash": + "sha256:b4e1c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4", + "enforcement_mode": "enforce", + }, + "data_class": "confidential", + "build_provenance": { + "slsa_level": 2, + "digest": "sha256:c9f7a5b2d4e6f8a0b2c4d6e8f0a2b4c6d8e0f2a4b6c8d0e2f4a6b8c0d2e4f6a8", + }, + "appraisal": appraisal, + "transparency": "https://scitt.example.org/receipts/abc123def456", + "cnf": { + "jwk": { + "kty": "EC", + "crv": "P-256", + "x": "dGhpcyBpcyBhIHRlc3QgeA", + "y": "dGhpcyBpcyBhIHRlc3QgeQ", + "kid": "tee-key-001", + } + }, + } + + +CANDIDATE_FIELD = "CANDIDATE:appraisal.policy_digest" +DEFERRED_PENDING = "agentrust-io/trace-spec#190" + +SPEC_DECIDED = ( + "agentrust-io/trace-spec docs/verification.md - a verifier records what it " + "actually resolved, and evidence that resolves and contradicts fails the " + "appraisal. Applied here to appraisal.policy_ref, which carries no digest." +) +SPEC_DEFERRED = ( + "agentrust-io/trace-spec docs/verification.md - evidence that does not " + "resolve downgrades honestly rather than failing. Which value records that " + "for appraisal.policy_ref is open: agentrust-io/trace-spec#190. This vector " + "asserts only what the outcome may NOT be." +) + + +def main(out_dir: Path | None = None) -> int: + """Write the set into *out_dir* (default: this directory). + + The parameter exists so the byte-reproduction guard can regenerate into a + temporary directory and compare, rather than overwriting the committed + files and comparing them to themselves — which would agree no matter what. + """ + here = Path(out_dir) if out_dir is not None else HERE + policies = here / "policies" + here.mkdir(parents=True, exist_ok=True) + policies.mkdir(exist_ok=True) + + # 1. Write the cited objects and digest their exact bytes. + digests: dict[str, str] = {} + raw: dict[str, bytes] = {} + for name, obj in POLICY_FILES.items(): + raw[name] = write_json(policies / name, obj) + digests[name] = sha256_of(raw[name]) + + d_v1 = digests["appraisal-policy-v1.json"] + d_rev2 = digests["appraisal-policy-v1-rev2.json"] + d_v2 = digests["appraisal-policy-v2.json"] + d_unrel = digests["unrelated-policy.json"] + + uri_v1 = BASE_URI + "appraisal-policy-v1.json" + uri_v2 = BASE_URI + "appraisal-policy-v2.json" + uri_withdrawn = BASE_URI + "appraisal-policy-withdrawn.json" + + def resolved(file_name: str, digest: str) -> dict[str, object]: + return { + "outcome": "resolved", + "file": f"policies/{file_name}", + "actual_sha256": digest, + } + + vectors: list[tuple[str, dict[str, object]]] = [ + ("01-no-binding-declared.json", { + "name": "no-binding-declared", + "description": ( + "The record cites an appraisal policy by bare URI and declares no " + "binding to what that URI held. This is every conformant record " + "today, so it must keep verifying: a set that rejected it would be " + "proposing a breaking change rather than describing a gap." + ), + "boundary": "accept", + "defect": "none - backward-compatibility control", + "spec": ( + "Mirrors the merged treatment of an undeclared depth in " + "agentrust-io/trace-spec#173, where a record that never declared is " + "read as surface rather than as a failure." + ), + "record": record_with(uri_v1), + "context": { + "cited_uri": uri_v1, + "resolution": { + "outcome": "not_attempted", + "note": "No binding is declared, so there is nothing to check.", + }, + "candidate_binding": None, + }, + "expected": { + "outcome": "pass", + "reason": "No binding declared; nothing to contradict.", + }, + }), + ("02-resolved-and-matches.json", { + "name": "resolved-and-matches", + "description": ( + "The cited URI resolves and its bytes hash to exactly the declared " + "candidate binding. The second must-accept vector, and the only one " + "in which a resolution actually succeeds." + ), + "boundary": "accept", + "defect": "none - positive control", + "spec": SPEC_DECIDED, + "record": record_with(uri_v1), + "context": { + "cited_uri": uri_v1, + "resolution": resolved("appraisal-policy-v1.json", d_v1), + "candidate_binding": {"field": CANDIDATE_FIELD, "value": d_v1}, + }, + "expected": { + "outcome": "pass", + "reason": "Declared binding equals the digest of what resolved.", + }, + }), + ("03-digest-mismatch-minimal-mutation.json", { + "name": "digest-mismatch-minimal-mutation", + "description": ( + "The cited URI now serves a document one character from the one " + "appraised: the SLSA floor moved from 2 to 3. The record still " + "declares the original digest. A verifier that compares only the " + "URI sees no change; a verifier that compares bytes sees the " + "substitution that flips this record's verdict." + ), + "boundary": "contradicted", + "defect": "cited object mutated minimally after appraisal", + "spec": SPEC_DECIDED, + "record": record_with(uri_v1), + "context": { + "cited_uri": uri_v1, + "resolution": resolved("appraisal-policy-v1-rev2.json", d_rev2), + "candidate_binding": {"field": CANDIDATE_FIELD, "value": d_v1}, + "note": ( + "policies/appraisal-policy-v1.json and " + "policies/appraisal-policy-v1-rev2.json differ in one character." + ), + }, + "expected": { + "outcome": "reject", + "reason": "Resolved bytes contradict the declared binding.", + }, + }), + ("04-digest-mismatch-different-object.json", { + "name": "digest-mismatch-different-object", + "description": ( + "The cited URI resolves to an unrelated policy document. Paired with " + "03 so the boundary is not carried by a single vector: a verifier " + "that only samples a prefix of the document, or compares lengths, " + "passes one of these two and fails the other." + ), + "boundary": "contradicted", + "defect": "cited object wholly replaced after appraisal", + "spec": SPEC_DECIDED, + "record": record_with(uri_v1), + "context": { + "cited_uri": uri_v1, + "resolution": resolved("unrelated-policy.json", d_unrel), + "candidate_binding": {"field": CANDIDATE_FIELD, "value": d_v1}, + }, + "expected": { + "outcome": "reject", + "reason": "Resolved bytes contradict the declared binding.", + }, + }), + ("05-referent-unreachable.json", { + "name": "referent-unreachable", + "description": ( + "The cited URI does not resolve; no object under policies/ " + "corresponds to it. Nothing was contradicted, because nothing was " + "read. This vector asserts only that the outcome is not affirming." + ), + "boundary": "unresolvable", + "defect": "referent unreachable at verification time", + "spec": SPEC_DEFERRED, + "record": record_with(uri_withdrawn), + "context": { + "cited_uri": uri_withdrawn, + "resolution": { + "outcome": "unreachable", + "file": None, + "note": "No sibling file corresponds to this URI, by construction.", + }, + "candidate_binding": {"field": CANDIDATE_FIELD, "value": d_v1}, + }, + "expected": { + "outcome": "deferred", + "deferred_pending": DEFERRED_PENDING, + "must_not": "affirming", + "reason": ( + "Reporting a check that was never performed as affirming is the " + "failure this set exists to name. Which value is reported " + "instead is not decided here." + ), + }, + }), + ("06-digest-algorithm-uncomputable.json", { + "name": "digest-algorithm-uncomputable", + "description": ( + "The referent resolves, but the declared binding names a digest " + "algorithm outside the set this profile's schema admits " + "(sha256 and sha384). The verifier cannot compute the comparison, " + "so it has not checked - which is distinct from having checked and " + "disagreed. Paired with 05: one cannot reach the object, the other " + "reaches it and cannot compute over it." + ), + "boundary": "unresolvable", + "defect": "digest algorithm the verifier cannot compute", + "spec": ( + SPEC_DEFERRED + + " Deliberately rhymes with the unverifiable / " + "digest_algorithm_unsupported classification PROPOSED in " + "agentrust-io/trace-spec#184 - an explicitly non-normative " + "draft, a surface facing this question rather than an " + "established answer to it - without adopting its vocabulary." + ), + "record": record_with(uri_v2), + "context": { + "cited_uri": uri_v2, + "resolution": resolved("appraisal-policy-v2.json", d_v2), + "candidate_binding": { + "field": CANDIDATE_FIELD, + "value": sha3_512_of(raw["appraisal-policy-v2.json"]), + "note": ( + "This is the correct SHA3-512 of the referent. It is " + "outside the schema's digest pattern " + "^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$, so no conformant " + "verifier is required to compute it - which is the only " + "reason this vector is undecided." + ), + }, + }, + "expected": { + "outcome": "deferred", + "deferred_pending": DEFERRED_PENDING, + "must_not": "affirming", + "reason": ( + "The comparison was not performed, so the result is not a " + "contradiction. Which value records that is not decided here." + ), + }, + }), + ("07-digest-bound-to-other-referent.json", { + "name": "digest-bound-to-other-referent", + "description": ( + "The declared binding is well formed and is the true digest of a " + "real object in this set - version 2 - while policy_ref cites " + "version 1. Both halves are individually valid and the pair is not. " + "A verifier that checks the digest is well formed, or that it " + "matches something it holds, passes this; only one that checks the " + "digest against what this URI resolved to rejects it." + ), + "boundary": "contradicted", + "defect": "binding well formed but bound to a different referent", + "spec": SPEC_DECIDED, + "record": record_with(uri_v1), + "context": { + "cited_uri": uri_v1, + "resolution": resolved("appraisal-policy-v1.json", d_v1), + "candidate_binding": { + "field": CANDIDATE_FIELD, + "value": d_v2, + "note": ( + "This is the digest of policies/appraisal-policy-v2.json, " + "which is not what cited_uri resolves to." + ), + }, + }, + "expected": { + "outcome": "reject", + "reason": ( + "The binding does not describe the object the record cites, " + "even though it describes some object." + ), + }, + }), + ] + + for filename, vector in vectors: + write_json(here / filename, vector) + + print(f"wrote {len(POLICY_FILES)} policy objects and {len(vectors)} vectors") + for name, digest in digests.items(): + print(f" policies/{name} {digest}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/vectors/appraisal-resolution/policies/appraisal-policy-v1-rev2.json b/tests/vectors/appraisal-resolution/policies/appraisal-policy-v1-rev2.json new file mode 100644 index 0000000..8259434 --- /dev/null +++ b/tests/vectors/appraisal-resolution/policies/appraisal-policy-v1-rev2.json @@ -0,0 +1,18 @@ +{ + "policy_id": "appraisal/baseline", + "version": "1.0.0", + "rules": [ + { + "claim": "runtime.platform", + "must_be_one_of": [ + "intel-tdx", + "amd-sev-snp", + "tpm2" + ] + }, + { + "claim": "build_provenance.slsa_level", + "minimum": 3 + } + ] +} diff --git a/tests/vectors/appraisal-resolution/policies/appraisal-policy-v1.json b/tests/vectors/appraisal-resolution/policies/appraisal-policy-v1.json new file mode 100644 index 0000000..f34a5b8 --- /dev/null +++ b/tests/vectors/appraisal-resolution/policies/appraisal-policy-v1.json @@ -0,0 +1,18 @@ +{ + "policy_id": "appraisal/baseline", + "version": "1.0.0", + "rules": [ + { + "claim": "runtime.platform", + "must_be_one_of": [ + "intel-tdx", + "amd-sev-snp", + "tpm2" + ] + }, + { + "claim": "build_provenance.slsa_level", + "minimum": 2 + } + ] +} diff --git a/tests/vectors/appraisal-resolution/policies/appraisal-policy-v2.json b/tests/vectors/appraisal-resolution/policies/appraisal-policy-v2.json new file mode 100644 index 0000000..164187a --- /dev/null +++ b/tests/vectors/appraisal-resolution/policies/appraisal-policy-v2.json @@ -0,0 +1,21 @@ +{ + "policy_id": "appraisal/baseline", + "version": "2.0.0", + "rules": [ + { + "claim": "runtime.platform", + "must_be_one_of": [ + "intel-tdx", + "amd-sev-snp" + ] + }, + { + "claim": "build_provenance.slsa_level", + "minimum": 3 + }, + { + "claim": "transparency", + "must_be_present": true + } + ] +} diff --git a/tests/vectors/appraisal-resolution/policies/unrelated-policy.json b/tests/vectors/appraisal-resolution/policies/unrelated-policy.json new file mode 100644 index 0000000..3b98d46 --- /dev/null +++ b/tests/vectors/appraisal-resolution/policies/unrelated-policy.json @@ -0,0 +1,13 @@ +{ + "policy_id": "retention/pii-90d", + "version": "1.4.2", + "rules": [ + { + "claim": "data_class", + "must_be_one_of": [ + "public", + "internal" + ] + } + ] +}