test(appraisal): candidate vectors for appraisal.policy_ref resolution

tests/test_appraisal_resolution.py

@@ -0,0 +1,202 @@
+"""The appraisal-resolution set proves itself: digests, referents, schema.
+
+No verifier resolves ``appraisal.policy_ref``, so there is no implementation to
+run these against. What can be checked — and is checked here — is that the set
+is internally what it claims to be:
+
+  * every record is valid under the packaged schema on this branch's base, so
+    the set describes conformant records rather than malformed ones;
+  * every declared digest really is, or really is not, the SHA-256 of the bytes
+    the vector says the citation resolved to, matching its expected outcome;
+  * the unreachable vector really has no resolvable object;
+  * vector 06's algorithm really is outside the digest set the schema admits.
+
+A vector whose expected outcome and whose bytes disagree is worse than no
+vector: it looks like coverage and argues for the wrong thing.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from pathlib import Path
+
+import jsonschema
+import pytest
+
+VECTOR_DIR = Path(__file__).parent / "vectors" / "appraisal-resolution"
+SCHEMA_DIGEST_PATTERN = re.compile(r"^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$")
+
+
+def _vector_paths() -> list[Path]:
+    return sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json"))
+
+
+def _load(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def _sha256_of_file(rel: str) -> str:
+    return "sha256:" + hashlib.sha256((VECTOR_DIR / rel).read_bytes()).hexdigest()
+
+
+VECTORS = _vector_paths()
+
+
+@pytest.mark.level0
+def test_the_set_has_the_seven_vectors_it_documents() -> None:
+    names = [p.name for p in VECTORS]
+    assert len(names) == 7, names
+    for n, path in enumerate(VECTORS, start=1):
+        assert path.name.startswith(f"{n:02d}-"), (
+            f"vectors must be contiguously numbered; got {path.name} at position {n}"
+        )
+
+
+@pytest.mark.level0
+@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem)
+def test_the_record_is_valid_under_the_packaged_schema(path: Path, schema) -> None:
+    """Schema validity, so the set is about resolution and not about malformed input."""
+    jsonschema.validate(_load(path)["record"], schema)
+
+
+@pytest.mark.level0
+@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem)
+def test_the_record_carries_a_bare_policy_ref_and_no_invented_field(path: Path) -> None:
+    """The candidate binding must never leak into the record.
+
+    ``appraisal`` is ``additionalProperties: false`` in the packaged schema, so
+    a record carrying the candidate field would be schema-invalid — and
+    proposing it is a schema decision this set does not make.
+    """
+    appraisal = _load(path)["record"]["appraisal"]
+    assert set(appraisal) <= {"status", "verifier", "policy_ref", "timestamp"}, (
+        f"record appraisal carries an unexpected key: {sorted(appraisal)}"
+    )
+    for key in appraisal:
+        assert "digest" not in key, f"record appraisal proposes a binding field: {key}"
+
+
+@pytest.mark.level0
+@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem)
+def test_the_declared_resolution_matches_the_bytes_on_disk(path: Path) -> None:
+    """Whatever the vector says resolution returned, the file must hash to it."""
+    ctx = _load(path)["context"]
+    resolution = ctx["resolution"]
+    if resolution["outcome"] != "resolved":
+        assert resolution.get("file") in (None,), (
+            "a resolution that did not resolve must not name a file"
+        )
+        return
+    rel = resolution["file"]
+    assert (VECTOR_DIR / rel).is_file(), f"{rel} is missing from the set"
+    assert resolution["actual_sha256"] == _sha256_of_file(rel), (
+        f"{rel} does not hash to the digest the vector records for it"
+    )
+
+
+@pytest.mark.level0
+@pytest.mark.negative
+@pytest.mark.parametrize("path", VECTORS, ids=lambda p: p.stem)
+def test_the_binding_agrees_with_the_expected_outcome(path: Path) -> None:
+    """The load-bearing self-proof.
+
+    accept  -> the declared binding equals the digest of what resolved
+               (or no binding is declared at all)
+    reject  -> a binding and a resolution both exist, and they disagree
+    deferred-> the comparison could not be performed at all
+    """
+    vector = _load(path)
+    ctx, outcome = vector["context"], vector["expected"]["outcome"]
+    binding = ctx.get("candidate_binding")
+    resolution = ctx["resolution"]
+    resolved = resolution["outcome"] == "resolved"
+
+    if outcome == "pass":
+        if binding is None:
+            # 01: nothing was declared, so there is nothing that could contradict.
+            assert resolution["outcome"] == "not_attempted", (
+                "a vector declaring no binding must not claim a resolution was "
+                "attempted; the point is that there was nothing to check"
+            )
+            return
+        assert resolved, "an accepting vector with a binding must have resolved"
+        assert binding["value"] == resolution["actual_sha256"], (
+            "accept requires the declared binding to equal what resolved"
+        )
+
+    elif outcome == "reject":
+        assert binding is not None, "a rejecting vector must declare a binding"
+        assert resolved, (
+            "a rejecting vector must have resolved something; otherwise nothing "
+            "was contradicted and the case is unresolvable, not contradicted"
+        )
+        assert SCHEMA_DIGEST_PATTERN.match(binding["value"]), (
+            "a rejecting vector's binding must be well formed, so the rejection "
+            "is about the referent and not about a malformed digest"
+        )
+        assert binding["value"] != resolution["actual_sha256"], (
+            "reject requires the declared binding to differ from what resolved"
+        )
+
+    elif outcome == "deferred":
+        computable = (
+            binding is not None
+            and SCHEMA_DIGEST_PATTERN.match(binding["value"]) is not None
+        )
+        assert not (resolved and computable), (
+            "a deferred vector must be one the verifier could not complete: "
+            "either the referent did not resolve, or the digest algorithm is "
+            "outside the set the schema admits"
+        )
+
+    else:  # pragma: no cover - guarded by the completeness test
+        raise AssertionError(f"unknown expected outcome {outcome!r}")
+
+
+@pytest.mark.level0
+@pytest.mark.negative
+def test_05_really_has_no_resolvable_object() -> None:
+    vector = _load(VECTOR_DIR / "05-referent-unreachable.json")
+    assert vector["context"]["resolution"]["outcome"] == "unreachable"
+    cited = vector["context"]["cited_uri"]
+    tail = cited.rsplit("/", 1)[-1]
+    assert not (VECTOR_DIR / "policies" / tail).exists(), (
+        "05 claims the referent is unreachable, but a sibling file matches its URI"
+    )
+
+
+@pytest.mark.level0
+@pytest.mark.negative
+def test_06_names_an_algorithm_the_schema_does_not_admit() -> None:
+    vector = _load(VECTOR_DIR / "06-digest-algorithm-uncomputable.json")
+    value = vector["context"]["candidate_binding"]["value"]
+    assert not SCHEMA_DIGEST_PATTERN.match(value), (
+        "06 claims the algorithm is uncomputable, but the digest matches the "
+        "pattern the schema admits"
+    )
+    assert vector["context"]["resolution"]["outcome"] == "resolved", (
+        "06 must reach its referent; that is what separates it from 05"
+    )
+
+
+@pytest.mark.level0
+def test_03_and_04_differ_in_kind_not_just_in_bytes() -> None:
+    """The pair that keeps the contradicted boundary off a single vector."""
+    minimal = _load(VECTOR_DIR / "03-digest-mismatch-minimal-mutation.json")
+    wholesale = _load(VECTOR_DIR / "04-digest-mismatch-different-object.json")
+
+    a = (VECTOR_DIR / minimal["context"]["resolution"]["file"]).read_bytes()
+    b = (VECTOR_DIR / "policies" / "appraisal-policy-v1.json").read_bytes()
+    assert len(a) == len(b), "03's mutation should not change the object's length"
+    assert sum(x != y for x, y in zip(a, b)) == 1, (
+        "03 is the minimal-mutation vector; it must differ from the appraised "
+        "object in exactly one byte"
+    )
+
+    c = (VECTOR_DIR / wholesale["context"]["resolution"]["file"]).read_bytes()
+    assert len(c) != len(b), (
+        "04 is the wholesale-substitution vector; a verifier comparing lengths "
+        "must be able to tell it from 03"
+    )

tests/test_appraisal_resolution_completeness.py

@@ -0,0 +1,194 @@
+"""Adequacy of the appraisal-resolution set, per the criteria on trace-spec#186.
+
+agentrust-io/trace-spec#186 (merged 2026-08-20) states what a conformance
+vector set is claiming: *a verifier that does not implement these rules will
+fail this set*. Three of its four criteria are checkable here and are checked;
+the fourth is about repository-wide bookkeeping and is noted below.
+
+It merged into trace-spec, where it grades that repository's ``examples/``.
+This repository has no adequacy harness, so nothing here is subject to it.
+These criteria are a standard this set was built to by choice, and the tests
+below are this set holding itself to them.
+
+    1. A set must fail BOTH unconditional implementations.
+       A set of all-rejections is passed by a verifier that rejects everything,
+       exactly as a set of all-acceptances is passed by one that accepts
+       everything. This set's three decided-reject vectors would, alone, be the
+       first failure. Vectors 01 and 02 exist to close it.
+    2. Every boundary needs more than one vector.
+       One vector cannot separate a check that reads a prefix from one that
+       reads the whole object.
+    3. Every set on disk is measured, or named with the test that measures it.
+       Repository-wide; trace-tests has no registry to add to, so it cannot be
+       asserted from inside one set. Recorded in the set's README instead.
+    4. Shortfalls are recorded exactly.
+       See KNOWN_SHORTFALLS below.
+
+These tests grade the set, not a verifier. No verifier resolves
+``appraisal.policy_ref`` today — that is the gap the set documents — so nothing
+here claims an implementation was exercised.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+
+import pytest
+
+VECTOR_DIR = Path(__file__).parent / "vectors" / "appraisal-resolution"
+
+DECIDED_OUTCOMES = {"pass", "reject"}
+ALL_OUTCOMES = DECIDED_OUTCOMES | {"deferred"}
+
+# Criterion 4: shortfalls asserted to their exact extent, so they cannot widen
+# quietly. Delete an entry when the shortfall is closed, not when it is excused.
+KNOWN_SHORTFALLS = {
+    "no_verifier_exercised": (
+        "No implementation resolves appraisal.policy_ref, so every expected "
+        "outcome is a claim about what a verifier should do, not a recording of "
+        "what one did. The set is a specification argument with runnable "
+        "internal consistency, not a conformance run."
+    ),
+    "unresolvable_outcome_unnamed": (
+        "Vectors 05 and 06 assert only that the outcome is not affirming. The "
+        "value a verifier should record is open on agentrust-io/trace-spec#190 "
+        "and is deliberately not proposed here."
+    ),
+}
+
+
+def _vectors() -> list[dict]:
+    out = []
+    for path in sorted(VECTOR_DIR.glob("[0-9][0-9]-*.json")):
+        out.append(json.loads(path.read_text(encoding="utf-8")))
+    return out
+
+
+def _outcome(vector: dict) -> str:
+    return vector["expected"]["outcome"]
+
+
+@pytest.mark.level0
+def test_the_set_is_not_empty() -> None:
+    assert len(_vectors()) == 7, "the set is seven vectors, 01 through 07"
+
+
+@pytest.mark.level0
+def test_criterion_1_the_set_fails_an_accept_everything_verifier() -> None:
+    """At least one vector a conformant verifier must reject."""
+    rejects = [v for v in _vectors() if _outcome(v) == "reject"]
+    assert rejects, (
+        "every vector expects acceptance, so a verifier that accepts "
+        "unconditionally passes the set"
+    )
+
+
+@pytest.mark.level0
+def test_criterion_1_the_set_fails_a_reject_everything_verifier() -> None:
+    """At least one vector a conformant verifier must accept.
+
+    This is the criterion the set would otherwise fail. Its subject is a family
+    of resolution failures, so every vector written from the motivating problem
+    alone is a reject or a deferral.
+    """
+    accepts = [v for v in _vectors() if _outcome(v) == "pass"]
+    assert accepts, (
+        "no vector expects acceptance, so a verifier that rejects "
+        "unconditionally passes the set"
+    )
+    assert len(accepts) >= 2, (
+        "one must-accept vector cannot separate a verifier that accepts only "
+        "records declaring no binding from one that also checks a matching "
+        "binding; 01 and 02 are that pair"
+    )
+
+
+@pytest.mark.level0
+def test_criterion_2_every_boundary_carries_at_least_two_vectors() -> None:
+    counts = Counter(v["boundary"] for v in _vectors())
+    thin = {b: n for b, n in counts.items() if n < 2}
+    assert not thin, f"boundaries carried by a single vector: {thin}"
+    assert set(counts) == {"accept", "contradicted", "unresolvable"}, (
+        f"unexpected boundary set: {sorted(counts)}"
+    )
+
+
+@pytest.mark.level0
+def test_no_two_vectors_share_a_defect() -> None:
+    defects = [v["defect"] for v in _vectors()]
+    dupes = [d for d, n in Counter(defects).items() if n > 1 and d != "none"]
+    assert not dupes, f"defect exercised by more than one vector: {dupes}"
+
+
+@pytest.mark.level0
+def test_every_expected_block_is_well_formed() -> None:
+    for v in _vectors():
+        name = v["name"]
+        exp = v["expected"]
+        assert exp["outcome"] in ALL_OUTCOMES, f"{name}: bad outcome {exp['outcome']!r}"
+        assert exp.get("reason"), f"{name}: expected block carries no reason"
+        if exp["outcome"] == "deferred":
+            assert exp.get("deferred_pending") == "agentrust-io/trace-spec#190", (
+                f"{name}: a deferred vector must name the issue it defers to"
+            )
+            assert exp.get("must_not") == "affirming", (
+                f"{name}: a deferred vector must still assert what it may not be"
+            )
+        else:
+            assert "deferred_pending" not in exp, (
+                f"{name}: a decided vector must not carry a deferral pointer"
+            )
+
+
+@pytest.mark.level0
+def test_no_vector_proposes_an_appraisal_status_value() -> None:
+    """The set must not coin the vocabulary trace-spec#190 exists to decide.
+
+    ``deferred`` is fixture bookkeeping. It must never appear as an
+    ``appraisal.status``, and no vector may assert a status for the
+    unresolvable case.
+    """
+    schema_enum = {"affirming", "warning", "contraindicated", "none"}
+    for v in _vectors():
+        status = v["record"]["appraisal"]["status"]
+        assert status in schema_enum, f"{v['name']}: invented status {status!r}"
+        exp = v["expected"]
+        assert exp["outcome"] not in schema_enum, (
+            f"{v['name']}: expected.outcome reuses an appraisal.status value, "
+            "which reads as proposing that value for this case"
+        )
+        if exp["outcome"] == "deferred":
+            assert "status" not in exp, (
+                f"{v['name']}: a deferred vector must not name a status to record"
+            )
+
+
+@pytest.mark.level0
+def test_candidate_binding_is_marked_candidate_everywhere_it_appears() -> None:
+    for v in _vectors():
+        binding = v["context"].get("candidate_binding")
+        if binding is None:
+            continue
+        assert binding["field"].startswith("CANDIDATE:"), (
+            f"{v['name']}: the binding field must be marked CANDIDATE, so it is "
+            "never read as a proposed schema field"
+        )
+
+
+@pytest.mark.level0
+def test_known_shortfalls_are_recorded_not_silent() -> None:
+    """Criterion 4: the gaps are asserted to their exact extent."""
+    assert set(KNOWN_SHORTFALLS) == {
+        "no_verifier_exercised",
+        "unresolvable_outcome_unnamed",
+    }, (
+        "the recorded shortfalls changed; update the README in the same commit "
+        "so the set never claims more coverage than it has"
+    )
+    deferred = [v for v in _vectors() if _outcome(v) == "deferred"]
+    assert len(deferred) == 2, (
+        "unresolvable_outcome_unnamed is recorded as covering exactly two "
+        f"vectors; found {len(deferred)}"
+    )