From 103fd8ecb05cf8feed3348c5fee52dd21544ca20 Mon Sep 17 00:00:00 2001 From: Siddhant Rai Date: Sun, 23 Aug 2026 15:17:24 +0530 Subject: [PATCH] feat(google-adk): add released-runner TRACE adapter Signed-off-by: Siddhant Rai --- .github/workflows/trace-adapters-tests.yml | 23 + README.md | 2 + integrations/google-adk/README.md | 94 ++++ .../google-adk/google_adk_to_trace.py | 484 ++++++++++++++++++ integrations/google-adk/integration.yaml | 22 + integrations/google-adk/requirements.txt | 1 + .../google-adk/test_google_adk_interop.py | 424 +++++++++++++++ .../google-adk/test_google_adk_to_trace.py | 435 ++++++++++++++++ marketplace/catalog.json | 24 +- noxfile.py | 14 +- 10 files changed, 1521 insertions(+), 2 deletions(-) create mode 100644 integrations/google-adk/README.md create mode 100644 integrations/google-adk/google_adk_to_trace.py create mode 100644 integrations/google-adk/integration.yaml create mode 100644 integrations/google-adk/requirements.txt create mode 100644 integrations/google-adk/test_google_adk_interop.py create mode 100644 integrations/google-adk/test_google_adk_to_trace.py diff --git a/.github/workflows/trace-adapters-tests.yml b/.github/workflows/trace-adapters-tests.yml index 80542e7..5174625 100644 --- a/.github/workflows/trace-adapters-tests.yml +++ b/.github/workflows/trace-adapters-tests.yml @@ -8,6 +8,7 @@ on: - "integrations/otel-genai/**" - "integrations/langchain/**" - "integrations/llamaindex/**" + - "integrations/google-adk/**" - "integrations/openshell/**" - ".github/workflows/trace-adapters-tests.yml" push: @@ -18,6 +19,7 @@ on: - "integrations/otel-genai/**" - "integrations/langchain/**" - "integrations/llamaindex/**" + - "integrations/google-adk/**" - "integrations/openshell/**" - ".github/workflows/trace-adapters-tests.yml" @@ -140,3 +142,24 @@ jobs: run: | pip install "langchain-core==1.6.0" "langgraph==1.2.11" python -m pytest integrations/langchain/test_langgraph_interop.py -q + + google-adk-adapter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + - name: Test evidence rules without Google ADK + run: | + pip install "agentrust-trace==0.9.0" pytest + python -m pytest integrations/google-adk/test_google_adk_to_trace.py -q + - name: Test released Google ADK interoperability and evidence rules + run: | + pip install "google-adk==2.7.1" "agentrust-trace-tests==0.5.0" + python -m pytest \ + integrations/google-adk/test_google_adk_to_trace.py \ + integrations/google-adk/test_google_adk_interop.py \ + -q diff --git a/README.md b/README.md index 76f557c..59be219 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ TRACE only works as a standard if it is genuinely neutral. Integrations are list | [Agent Passport System](integrations/aeoess-aps/) | aeoess | trace | community | | [comply54](integrations/comply54/) | comply54 | trace | community | | [DecisionAssure](integrations/decisionassure/) | DecisionAssure (a1k7) | trace | community | +| [Google ADK](integrations/google-adk/) | agentrust-io | trace | community | | [LangChain](integrations/langchain/) | agentrust-io | trace | community | | [LlamaIndex](integrations/llamaindex/) | agentrust-io | trace | community | | [Nobulex](integrations/nobulex/) | Nobulex | trace | community | @@ -57,6 +58,7 @@ TRACE only works as a standard if it is genuinely neutral. Integrations are list | Framework | Adapter | Released framework exercised in CI | Evidence boundary | |---|---|---|---| +| Google ADK | [Google ADK](integrations/google-adk/) | Yes - Google ADK 2.7.1 `InMemoryRunner` plugin lifecycle | Callback-visible invocation, model, and available tool identity; no payloads, retries, agent graph, function-body execution, or policy enforcement | | LangChain | [LangChain](integrations/langchain/) | Yes — LangChain Core 1.6.0 callback contract | Tool identity and outcome plus model identity; no chain topology or runnable state | | LangGraph | [LangChain](integrations/langchain/) | Yes — LangGraph 1.2.11 `StateGraph` with a nested tool call | Propagated tool callbacks; no nodes, edges, state transitions, checkpoints, or rollback decisions | | LlamaIndex | [LlamaIndex](integrations/llamaindex/) | No — current tests use representative event objects | Allow-listed tool and model fields; released-framework interoperability remains unverified | diff --git a/integrations/google-adk/README.md b/integrations/google-adk/README.md new file mode 100644 index 0000000..f81de89 --- /dev/null +++ b/integrations/google-adk/README.md @@ -0,0 +1,94 @@ +# Google ADK to TRACE + +Emits a TRACE v0.2 Trust Record from the released Google ADK `BasePlugin` +lifecycle. CI exercises Google ADK 2.7.1 through a real `InMemoryRunner` with a +deterministic local model, so the interoperability test makes no network call. + +## Evidence boundary + +The plugin runs inside the operator's ADK runner. Its records are first-party +evidence and carry no `origin` block. Without a hardware attestation they use +`runtime.platform: software-only`; building a record does not appraise it, so +`appraisal.status` is `none`. + +| Observed from ADK | Supplied by the operator | Not claimed | +|---|---|---| +| Invocation id, model id, available tool name, function-call fingerprint, callback-visible lifecycle outcome | Model provider, workload identity and digest, policy bytes, data class | Prompts, responses, arguments, results, exception text, retries, agent graph, policy enforcement, whether a tool function body ran | + +The model provider is operator-supplied because ADK can use models from more +than one provider. A single TRACE record has one model field, so an invocation +that exposes multiple model ids is refused rather than relabelled. A caller may +supply the model id only when ADK exposed none; it may not override an observed +id. + +The plugin starts each run as `incomplete`. A successful `after_run` changes it +to `ok`; a reported run error changes it to `error`. A cancelled or interrupted +run that never reaches either callback remains `incomplete`. That is lifecycle +evidence, not a claim that ADK reported a cancellation reason. + +Tool starts are retained in call order and correlated with completions through +ADK's function-call id. An unmatched start remains `incomplete`; an unmatched +completion is retained with `observed_start: false`. Neither case is silently +dropped. When an id is absent and more than one same-name start is pending, the +completion is retained as uncorrelated instead of being assigned by FIFO. + +These are callback-visible outcomes. Another plugin can short-circuit a tool or +recover its exception. The adapter therefore does not claim that the function +body ran. If an error callback is followed by a recovered completion for the +same fingerprint, both outcomes are retained on one call and the final +callback-visible outcome is `ok`. + +## Use it + +```bash +pip install agentrust-trace google-adk==2.7.1 +``` + +```python +from agentrust_trace.sign import generate_key, sign_record +from google.adk.apps import App +from google.adk.runners import InMemoryRunner +from google_adk_to_trace import GoogleAdkTracePlugin + +plugin = GoogleAdkTracePlugin() +app = App(name="research_app", root_agent=agent, plugins=[plugin]) +runner = InMemoryRunner(app=app) +await runner.run_debug("...", quiet=True) + +invocation_id = plugin.invocation_ids[-1] +record = plugin.build_record( + invocation_id, + subject="spiffe://example.org/agent/research-bot", + policy_bundle=open("policy.cedar", "rb").read(), + workload_digest="sha256:...", + data_class="internal", + model_provider="google", +) +signed = sign_record(record, generate_key()) +plugin.discard(invocation_id) +``` + +`enforcement_mode` defaults to `declared`: the policy is bound into the signed +record, but Google ADK itself did not evaluate it. Override that value only when +a separate enforcement layer actually evaluated the policy. + +One plugin can observe concurrent invocations. It retains state by ADK +invocation id until `discard()` is called, so long-running processes should +discard an invocation after persisting its record. + +## Tests + +```bash +pip install -r requirements.txt pytest +python -m pytest test_google_adk_to_trace.py -q + +pip install google-adk==2.7.1 agentrust-trace-tests==0.5.0 +python -m pytest test_google_adk_interop.py -q +``` + +The first suite exercises evidence construction without installing ADK. The +second uses the released runner and checks success, tool failure, cancellation, +concurrent invocations, payload exclusion, signed TRACE validation, and Level 0 +conformance for the optional externally enforced path. The bare ADK path keeps +the honest `declared` policy mode; `agentrust-trace-tests` 0.5.0 predates that +mode, so the conformance fixture uses `advisory` to represent an external layer. diff --git a/integrations/google-adk/google_adk_to_trace.py b/integrations/google-adk/google_adk_to_trace.py new file mode 100644 index 0000000..5f838fa --- /dev/null +++ b/integrations/google-adk/google_adk_to_trace.py @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 +"""Google ADK plugin -> TRACE v0.2 Trust Record. + +The plugin observes the released Google ADK callback lifecycle from inside the +runner. It records only invocation identity, model identity, available tool +identity, and callback-visible lifecycle outcomes. User content, model contents +and responses, tool arguments and results, and exception messages are +deliberately never read into evidence. + +Google ADK does not evaluate a TRACE policy. Records therefore default to +``policy.enforcement_mode: declared`` and ``appraisal.status: none``. They have +no ``origin`` block because this is first-party, in-process observation. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import threading +import time +from dataclasses import dataclass, field, replace +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from google.adk.plugins.base_plugin import BasePlugin as _GoogleAdkBasePlugin +else: + try: + from google.adk.plugins.base_plugin import BasePlugin as _GoogleAdkBasePlugin + except ModuleNotFoundError as exc: + if exc.name not in {"google", "google.adk", "google.adk.plugins"}: + raise + + class _GoogleAdkBasePlugin: + """Fallback for framework-free evidence tests.""" + + def __init__(self, name: str) -> None: + self.name = name + + +__all__ = ["GoogleAdkTracePlugin", "MissingEvidence", "ToolCall", "build_record"] + +_DIGEST_PREFIX = "sha256:" +TRACE_PROFILE = "tag:agentrust-io.com,2026:trace-v0.2" +ENFORCEMENT_MODES = ("enforce", "advisory", "silent", "declared") + + +class MissingEvidence(ValueError): + """Raised rather than inventing a required TRACE claim.""" + + +@dataclass(frozen=True) +class ToolCall: + """One allow-listed tool observation, without arguments or results.""" + + name: str | None + function_call_fingerprint: str | None + outcome: str # "ok" | "error" | "incomplete" + observed_start: bool = True + observed_outcomes: tuple[str, ...] = () + + +@dataclass +class _ObservedInvocation: + model_ids: set[str] = field(default_factory=set) + tools: list[ToolCall] = field(default_factory=list) + pending: dict[str, list[int]] = field(default_factory=dict) + finished_by_id: dict[str, int] = field(default_factory=dict) + uncorrelated_completions: list[ToolCall] = field(default_factory=list) + outcome: str = "incomplete" + root_agent_name: str | None = None + root_completed: bool = False + + +def _nonempty_string(value: Any) -> str | None: + return value if isinstance(value, str) and value else None + + +def _invocation_id(context: Any) -> str: + value = _nonempty_string(getattr(context, "invocation_id", None)) + if value is None: + raise MissingEvidence("Google ADK did not expose a non-empty invocation_id.") + return value + + +def _tool_name(tool: Any) -> str | None: + return _nonempty_string(getattr(tool, "name", None)) + + +def _function_call_fingerprint(tool_context: Any) -> str | None: + raw_id = _nonempty_string(getattr(tool_context, "function_call_id", None)) + if raw_id is None: + return None + return _DIGEST_PREFIX + hashlib.sha256(raw_id.encode()).hexdigest() + + +def _pending_key(function_call_fingerprint: str | None, name: str | None) -> str: + if function_call_fingerprint is not None: + return "id:" + function_call_fingerprint + return "name:" + (name or "") + + +class GoogleAdkTracePlugin(_GoogleAdkBasePlugin): + """Observe independent Google ADK invocations through ``BasePlugin``. + + One plugin may be shared by concurrent runner invocations. State is keyed + by ADK's invocation id and tool lifecycle events are correlated by the + function-call id when the SDK supplies one. + + Every callback is observational. Callbacks that may replace framework + values explicitly return ``None`` so execution is never short-circuited. + """ + + def __init__(self, name: str = "agentrust_trace") -> None: + super().__init__(name=name) + self._observed: dict[str, _ObservedInvocation] = {} + self._lock = threading.RLock() + + async def before_run_callback(self, *, invocation_context: Any) -> None: + invocation_id = _invocation_id(invocation_context) + root_agent = getattr(invocation_context, "agent", None) + with self._lock: + self._observed[invocation_id] = _ObservedInvocation( + root_agent_name=_nonempty_string(getattr(root_agent, "name", None)) + ) + return None + + async def after_run_callback(self, *, invocation_context: Any) -> None: + invocation_id = _invocation_id(invocation_context) + with self._lock: + state = self._observed.setdefault(invocation_id, _ObservedInvocation()) + if state.outcome != "error" and state.root_completed: + state.outcome = "ok" + return None + + async def before_agent_callback(self, *, agent: Any, callback_context: Any) -> None: # noqa: ARG002 + return None + + async def after_agent_callback(self, *, agent: Any, callback_context: Any) -> None: + invocation_id = _invocation_id(callback_context) + agent_name = _nonempty_string(getattr(agent, "name", None)) + parent_agent = getattr(agent, "parent_agent", None) + with self._lock: + state = self._observed.setdefault(invocation_id, _ObservedInvocation()) + if ( + parent_agent is None + and agent_name is not None + and agent_name == state.root_agent_name + ): + state.root_completed = True + return None + + async def on_run_error_callback( + self, + *, + invocation_context: Any, + error: Exception, # noqa: ARG002 - exception payload is excluded + ) -> None: + self._mark_run_error(_invocation_id(invocation_context)) + return None + + async def on_agent_error_callback( + self, + *, + agent: Any, # noqa: ARG002 + callback_context: Any, # noqa: ARG002 - later plugins may recover + error: Exception, # noqa: ARG002 + ) -> None: + return None + + async def before_model_callback( + self, *, callback_context: Any, llm_request: Any + ) -> None: + invocation_id = _invocation_id(callback_context) + model_id = _nonempty_string(getattr(llm_request, "model", None)) + if model_id is not None: + with self._lock: + state = self._observed.setdefault(invocation_id, _ObservedInvocation()) + state.model_ids.add(model_id) + return None + + async def after_model_callback( + self, + *, + callback_context: Any, # noqa: ARG002 - no response fields are evidence + llm_response: Any, # noqa: ARG002 - model payload is excluded + ) -> None: + return None + + async def on_model_error_callback( + self, + *, + callback_context: Any, # noqa: ARG002 - later plugins may recover + llm_request: Any, # noqa: ARG002 + error: Exception, # noqa: ARG002 + ) -> None: + return None + + async def before_tool_callback( + self, + *, + tool: Any, + tool_args: dict[str, Any], # noqa: ARG002 + tool_context: Any, + ) -> None: + invocation_id = _invocation_id(tool_context) + name = _tool_name(tool) + function_call_fingerprint = _function_call_fingerprint(tool_context) + call = ToolCall( + name=name, + function_call_fingerprint=function_call_fingerprint, + outcome="incomplete", + ) + with self._lock: + state = self._observed.setdefault(invocation_id, _ObservedInvocation()) + index = len(state.tools) + state.tools.append(call) + state.pending.setdefault( + _pending_key(function_call_fingerprint, name), [] + ).append(index) + return None + + async def after_tool_callback( + self, + *, + tool: Any, + tool_args: dict[str, Any], # noqa: ARG002 + tool_context: Any, + result: dict[str, Any], # noqa: ARG002 + ) -> None: + self._finish_tool(tool, tool_context, "ok") + return None + + async def on_tool_error_callback( + self, + *, + tool: Any, + tool_args: dict[str, Any], # noqa: ARG002 + tool_context: Any, + error: Exception, # noqa: ARG002 + ) -> None: + self._finish_tool(tool, tool_context, "error") + return None + + def _mark_run_error(self, invocation_id: str) -> None: + with self._lock: + state = self._observed.setdefault(invocation_id, _ObservedInvocation()) + state.outcome = "error" + + def _finish_tool(self, tool: Any, tool_context: Any, outcome: str) -> None: + invocation_id = _invocation_id(tool_context) + name = _tool_name(tool) + function_call_fingerprint = _function_call_fingerprint(tool_context) + key = _pending_key(function_call_fingerprint, name) + with self._lock: + state = self._observed.setdefault(invocation_id, _ObservedInvocation()) + indexes = state.pending.get(key) + if indexes and len(indexes) == 1: + index = indexes.pop(0) + state.pending.pop(key, None) + call = state.tools[index] + state.tools[index] = replace( + call, + outcome=outcome, + observed_outcomes=call.observed_outcomes + (outcome,), + ) + if function_call_fingerprint is not None: + state.finished_by_id[key] = index + return + + if indexes: + # More than one start has the same correlation key. Assigning + # this completion by FIFO would invent which call finished. + state.uncorrelated_completions.append( + ToolCall( + name=name, + function_call_fingerprint=function_call_fingerprint, + outcome=outcome, + observed_start=False, + observed_outcomes=(outcome,), + ) + ) + return + + if function_call_fingerprint is not None and key in state.finished_by_id: + index = state.finished_by_id[key] + call = state.tools[index] + state.tools[index] = replace( + call, + outcome=outcome, + observed_outcomes=call.observed_outcomes + (outcome,), + ) + return + + # A plugin attached after the start still records the completion. + state.tools.append( + ToolCall( + name=name, + function_call_fingerprint=function_call_fingerprint, + outcome=outcome, + observed_start=False, + observed_outcomes=(outcome,), + ) + ) + + @property + def invocation_ids(self) -> list[str]: + with self._lock: + return list(self._observed) + + def tool_calls(self, invocation_id: str) -> list[ToolCall]: + with self._lock: + return list(self._require_invocation(invocation_id).tools) + + def transcript_bytes(self, invocation_id: str) -> bytes: + """Canonical lifecycle evidence over an allow-list of non-payload fields.""" + with self._lock: + state = self._require_invocation(invocation_id) + body = { + "invocation_id": invocation_id, + "outcome": state.outcome, + "tools": [ + { + "function_call_fingerprint": call.function_call_fingerprint, + "observed_start": call.observed_start, + "observed_outcomes": list(call.observed_outcomes), + "outcome": call.outcome, + "tool": call.name, + } + for call in state.tools + ], + "uncorrelated_completions": [ + { + "function_call_fingerprint": call.function_call_fingerprint, + "observed_outcomes": list(call.observed_outcomes), + "outcome": call.outcome, + "tool": call.name, + } + for call in state.uncorrelated_completions + ], + } + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode() + + def discard(self, invocation_id: str) -> bool: + """Remove retained evidence after the caller has persisted its record.""" + with self._lock: + return self._observed.pop(invocation_id, None) is not None + + def build_record( + self, + invocation_id: str, + *, + subject: str, + policy_bundle: bytes, + enforcement_mode: str = "declared", + workload_digest: str, + data_class: str, + model_provider: str | None = None, + model_id: str | None = None, + attestation: dict[str, str] | None = None, + iat: int | None = None, + ) -> dict[str, Any]: + """Build an unsigned record for one retained ADK invocation.""" + with self._lock: + state = self._require_invocation(invocation_id) + if len(state.model_ids) > 1: + raise MissingEvidence( + "the invocation used multiple model ids; one TRACE record cannot " + "truthfully relabel them as a single model" + ) + observed_model = ( + next(iter(state.model_ids)) if len(state.model_ids) == 1 else None + ) + if observed_model is not None and model_id not in {None, observed_model}: + raise MissingEvidence( + f"model_id {model_id!r} conflicts with observed model " + f"{observed_model!r}" + ) + transcript = self.transcript_bytes(invocation_id) + tool_count = len(state.tools) + return build_record( + subject=subject, + policy_bundle=policy_bundle, + enforcement_mode=enforcement_mode, + workload_digest=workload_digest, + data_class=data_class, + model_provider=model_provider, + model_id=observed_model or model_id, + transcript=transcript, + tool_count=tool_count, + attestation=attestation, + iat=iat, + ) + + def _require_invocation(self, invocation_id: str) -> _ObservedInvocation: + try: + return self._observed[invocation_id] + except KeyError as exc: + raise MissingEvidence( + f"no evidence retained for invocation {invocation_id!r}" + ) from exc + + +def _digest(data: bytes) -> str: + return _DIGEST_PREFIX + hashlib.sha256(data).hexdigest() + + +def build_record( + *, + subject: str, + policy_bundle: bytes, + enforcement_mode: str = "declared", + workload_digest: str, + data_class: str, + model_provider: str | None, + model_id: str | None, + transcript: bytes, + tool_count: int, + attestation: dict[str, str] | None = None, + iat: int | None = None, +) -> dict[str, Any]: + """Construct a TRACE record without importing Google ADK.""" + if not re.match(r"^(spiffe://[^/]+/.+|did:[a-z0-9]+:.+)$", subject or ""): + raise MissingEvidence( + f"subject {subject!r} must be a SPIFFE URI or a DID; " + "the adapter may not invent identity." + ) + if not policy_bundle: + raise MissingEvidence( + "policy_bundle must contain the policy bytes being declared." + ) + if enforcement_mode not in ENFORCEMENT_MODES: + raise MissingEvidence( + f"enforcement_mode must be one of {', '.join(ENFORCEMENT_MODES)}." + ) + if not model_provider or not model_id: + raise MissingEvidence( + "model_provider and model_id are required. Google ADK exposes the " + "model id, but its model can come from different providers, so the " + "adapter will not guess." + ) + if not re.match(r"^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$", workload_digest or ""): + raise MissingEvidence( + "workload_digest must be a sha256:/sha384: artifact digest." + ) + + if attestation: + platform = attestation.get("platform") + measurement = attestation.get("measurement", "") + if platform == "software-only": + raise MissingEvidence("omit attestation for a software-only run") + if not platform or not re.match( + r"^sha(256:[0-9a-f]{64}|384:[0-9a-f]{96})$", measurement + ): + raise MissingEvidence("attestation requires a platform and measured digest") + runtime = {"platform": platform, "measurement": measurement} + else: + runtime = { + "platform": "software-only", + "measurement": _digest( + workload_digest.encode() + b"\n" + _digest(policy_bundle).encode() + ), + } + + record: dict[str, Any] = { + "eat_profile": TRACE_PROFILE, + "iat": int(iat if iat is not None else time.time()), + "subject": subject, + "model": {"provider": model_provider, "model_id": model_id}, + "runtime": runtime, + "policy": { + "bundle_hash": _digest(policy_bundle), + "enforcement_mode": enforcement_mode, + }, + "data_class": data_class, + "build_provenance": {"slsa_level": 0, "digest": workload_digest}, + "appraisal": {"status": "none", "verifier": "google-adk-adapter"}, + } + if tool_count: + record["tool_transcript"] = { + "hash": _digest(transcript), + "call_count": tool_count, + } + return record diff --git a/integrations/google-adk/integration.yaml b/integrations/google-adk/integration.yaml new file mode 100644 index 0000000..da93f79 --- /dev/null +++ b/integrations/google-adk/integration.yaml @@ -0,0 +1,22 @@ +name: Google ADK +vendor: agentrust-io +integrates_with: + - trace +description: >- + Emits a TRACE Trust Record from Google ADK callbacks using callback-visible + tool and model identity without payloads. +maintainer: + github: imran-siddique +repository: https://github.com/agentrust-io/integrations +license: Apache-2.0 +tier: community +marketplace: + category: Frameworks + mark: ADK + keywords: [callback, framework, tools] +trace_roles: + - record-producer +trace_conformance_level: 0 +tested_against: + agentrust-trace: "0.9.0" + agentrust-trace-tests: "0.5.0" diff --git a/integrations/google-adk/requirements.txt b/integrations/google-adk/requirements.txt new file mode 100644 index 0000000..df700ea --- /dev/null +++ b/integrations/google-adk/requirements.txt @@ -0,0 +1 @@ +agentrust-trace>=0.9.0 diff --git a/integrations/google-adk/test_google_adk_interop.py b/integrations/google-adk/test_google_adk_interop.py new file mode 100644 index 0000000..b996665 --- /dev/null +++ b/integrations/google-adk/test_google_adk_interop.py @@ -0,0 +1,424 @@ +"""Released Google ADK 2.7.1 Runner interoperability regression.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import pathlib +import sys +from collections.abc import AsyncGenerator + +import pytest +from agentrust_trace.models import TrustRecord +from agentrust_trace.sign import generate_key, sign_record +from google.adk.agents import LlmAgent, SequentialAgent +from google.adk.apps import App +from google.adk.models import BaseLlm, LlmRequest, LlmResponse +from google.adk.plugins.base_plugin import BasePlugin +from google.adk.runners import InMemoryRunner +from google.genai import types +from pydantic import PrivateAttr +from trace_tests import runner as trace_tests_runner +from trace_tests.result import Status + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +from google_adk_to_trace import GoogleAdkTracePlugin + +SECRET = "customer-account-reference-that-must-not-enter-the-record" +DIGEST = "sha256:" + "e" * 64 + + +class SequencedModel(BaseLlm): + model: str = "released-adk-test-model" + responses: list[LlmResponse] + response_index: int = 0 + + async def generate_content_async( + self, + llm_request: LlmRequest, # noqa: ARG002 - deterministic fixture + stream: bool = False, # noqa: ARG002 + ) -> AsyncGenerator[LlmResponse, None]: + response = self.responses[self.response_index] + self.response_index += 1 + yield response + + +class BlockingModel(BaseLlm): + model: str = "released-adk-blocking-model" + _started: asyncio.Event = PrivateAttr(default_factory=asyncio.Event) + + @property + def started(self) -> asyncio.Event: + return self._started + + async def generate_content_async( + self, + llm_request: LlmRequest, # noqa: ARG002 - deterministic fixture + stream: bool = False, # noqa: ARG002 + ) -> AsyncGenerator[LlmResponse, None]: + self._started.set() + await asyncio.Event().wait() + yield LlmResponse( + content=types.Content(role="model", parts=[types.Part(text="unused")]) + ) + + +class RecoverToolErrorPlugin(BasePlugin): + def __init__(self) -> None: + super().__init__(name="recover_tool_error") + + async def on_tool_error_callback( + self, + *, + tool, # noqa: ARG002 - recovery fixture + tool_args, # noqa: ARG002 - recovery fixture + tool_context, # noqa: ARG002 - recovery fixture + error, # noqa: ARG002 - recovery fixture + ) -> dict[str, bool]: + return {"recovered": True} + + +class ShortCircuitToolPlugin(BasePlugin): + def __init__(self) -> None: + super().__init__(name="short_circuit_tool") + + async def before_tool_callback( + self, + *, + tool, # noqa: ARG002 - short-circuit fixture + tool_args, # noqa: ARG002 - short-circuit fixture + tool_context, # noqa: ARG002 - short-circuit fixture + ) -> dict[str, bool]: + return {"synthetic": True} + + +def model_for_tool( + tool_name: str, *, arguments: dict[str, str], call_id: str = "call-1" +) -> SequencedModel: + return SequencedModel( + responses=[ + LlmResponse( + content=types.Content( + role="model", + parts=[ + types.Part( + function_call=types.FunctionCall( + id=call_id, + name=tool_name, + args=arguments, + ) + ) + ], + ) + ), + LlmResponse( + content=types.Content(role="model", parts=[types.Part(text="complete")]) + ), + ] + ) + + +def text_model(text: str = "complete") -> SequencedModel: + return SequencedModel( + responses=[ + LlmResponse( + content=types.Content(role="model", parts=[types.Part(text=text)]) + ) + ] + ) + + +def build_signed(plugin: GoogleAdkTracePlugin, invocation_id: str) -> dict: + return sign_record( + plugin.build_record( + invocation_id, + subject="spiffe://example.org/agent/google-adk", + policy_bundle=b'{"rules":["no-payload-egress"]}', + workload_digest=DIGEST, + data_class="confidential", + model_provider="test-provider", + iat=1_700_000_000, + ), + generate_key(), + ) + + +def runner_for( + agent: LlmAgent, plugin: GoogleAdkTracePlugin, name: str +) -> InMemoryRunner: + return InMemoryRunner(app=App(name=name, root_agent=agent, plugins=[plugin])) + + +def measure_payload(payload: str) -> dict[str, int]: + """Return the length without returning the input.""" + return {"length": len(payload)} + + +def reject_payload(payload: str) -> dict[str, str]: + """Raise a deterministic tool failure.""" + raise RuntimeError(f"rejected {payload}") + + +def must_not_execute(payload: str) -> dict[str, str]: + """Fail if a short-circuited tool reaches its function body.""" + raise AssertionError(f"tool unexpectedly executed with {payload}") + + +def test_released_runner_success_emits_valid_level_zero_record() -> None: + plugin = GoogleAdkTracePlugin() + agent = LlmAgent( + name="success_agent", + model=model_for_tool("measure_payload", arguments={"payload": SECRET}), + tools=[measure_payload], + ) + runner = runner_for(agent, plugin, "success_app") + + asyncio.run(runner.run_debug(SECRET, quiet=True)) + + assert isinstance(plugin, BasePlugin) + assert len(plugin.invocation_ids) == 1 + invocation_id = plugin.invocation_ids[0] + calls = plugin.tool_calls(invocation_id) + assert [(call.name, call.outcome) for call in calls] == [("measure_payload", "ok")] + assert calls[0].function_call_fingerprint == ( + "sha256:" + hashlib.sha256(b"call-1").hexdigest() + ) + transcript = json.loads(plugin.transcript_bytes(invocation_id)) + assert transcript["outcome"] == "ok" + assert SECRET.encode() not in plugin.transcript_bytes(invocation_id) + + signed = build_signed(plugin, invocation_id) + assert SECRET not in str(signed) + parsed = TrustRecord.model_validate(signed) + assert parsed.runtime.platform == "software-only" + assert parsed.tool_transcript is not None + assert parsed.tool_transcript.call_count == 1 + assert parsed.appraisal.status == "none" + assert "origin" not in signed + assert "transparency" not in signed + + +def test_released_runner_model_only_run_omits_tool_transcript() -> None: + plugin = GoogleAdkTracePlugin() + agent = LlmAgent(name="model_only_agent", model=text_model()) + runner = runner_for(agent, plugin, "model_only_app") + + asyncio.run(runner.run_debug("test", quiet=True)) + + invocation_id = plugin.invocation_ids[0] + assert plugin.tool_calls(invocation_id) == [] + assert json.loads(plugin.transcript_bytes(invocation_id))["outcome"] == "ok" + + signed = build_signed(plugin, invocation_id) + parsed = TrustRecord.model_validate(signed) + assert parsed.model.model_id == "released-adk-test-model" + assert parsed.tool_transcript is None + + +def test_model_supplied_call_id_is_fingerprinted_before_retention() -> None: + plugin = GoogleAdkTracePlugin() + agent = LlmAgent( + name="secret_id_agent", + model=model_for_tool( + "measure_payload", + arguments={"payload": "safe"}, + call_id=SECRET, + ), + tools=[measure_payload], + ) + runner = runner_for(agent, plugin, "secret_id_app") + + asyncio.run(runner.run_debug("test", quiet=True)) + + invocation_id = plugin.invocation_ids[0] + transcript = plugin.transcript_bytes(invocation_id) + assert SECRET.encode() not in transcript + assert plugin.tool_calls(invocation_id)[0].function_call_fingerprint == ( + "sha256:" + hashlib.sha256(SECRET.encode()).hexdigest() + ) + + +def test_level_zero_conformance_with_external_advisory_enforcement() -> None: + """The optional externally enforced path passes the released Level 0 suite.""" + plugin = GoogleAdkTracePlugin() + agent = LlmAgent( + name="conformance_agent", + model=model_for_tool("measure_payload", arguments={"payload": "test"}), + tools=[measure_payload], + ) + runner = runner_for(agent, plugin, "conformance_app") + asyncio.run(runner.run_debug("test", quiet=True)) + invocation_id = plugin.invocation_ids[0] + signed = sign_record( + plugin.build_record( + invocation_id, + subject="spiffe://example.org/agent/google-adk", + policy_bundle=b'{"rules":["external-advisory-layer"]}', + enforcement_mode="advisory", + workload_digest=DIGEST, + data_class="internal", + model_provider="test-provider", + ), + generate_key(), + ) + + findings = [ + finding + for module_findings in trace_tests_runner.run(signed, "trace", 0).values() + for finding in module_findings + ] + assert [finding.code for finding in findings if finding.status is Status.FAIL] == [] + + +def test_recovered_tool_error_remains_one_correlated_call() -> None: + plugin = GoogleAdkTracePlugin() + agent = LlmAgent( + name="recovery_agent", + model=model_for_tool("reject_payload", arguments={"payload": SECRET}), + tools=[reject_payload], + ) + app = App( + name="recovery_app", + root_agent=agent, + plugins=[plugin, RecoverToolErrorPlugin()], + ) + runner = InMemoryRunner(app=app) + + asyncio.run(runner.run_debug("recover", quiet=True)) + + invocation_id = plugin.invocation_ids[0] + calls = plugin.tool_calls(invocation_id) + assert len(calls) == 1 + assert calls[0].outcome == "ok" + assert calls[0].observed_outcomes == ("error", "ok") + assert SECRET.encode() not in plugin.transcript_bytes(invocation_id) + + +def test_earlier_short_circuit_is_recorded_as_completion_without_start() -> None: + plugin = GoogleAdkTracePlugin() + agent = LlmAgent( + name="short_circuit_agent", + model=model_for_tool("must_not_execute", arguments={"payload": SECRET}), + tools=[must_not_execute], + ) + app = App( + name="short_circuit_app", + root_agent=agent, + plugins=[ShortCircuitToolPlugin(), plugin], + ) + runner = InMemoryRunner(app=app) + + asyncio.run(runner.run_debug("short circuit", quiet=True)) + + invocation_id = plugin.invocation_ids[0] + calls = plugin.tool_calls(invocation_id) + assert len(calls) == 1 + assert (calls[0].observed_start, calls[0].outcome) == (False, "ok") + assert SECRET.encode() not in plugin.transcript_bytes(invocation_id) + + +def test_released_runner_reports_tool_failure_without_exception_payload() -> None: + plugin = GoogleAdkTracePlugin() + agent = LlmAgent( + name="failure_agent", + model=model_for_tool("reject_payload", arguments={"payload": SECRET}), + tools=[reject_payload], + ) + runner = runner_for(agent, plugin, "failure_app") + + with pytest.raises(RuntimeError, match="rejected"): + asyncio.run(runner.run_debug("fail", quiet=True)) + + invocation_id = plugin.invocation_ids[0] + assert plugin.tool_calls(invocation_id)[0].outcome == "error" + assert b'"outcome":"error"' in plugin.transcript_bytes(invocation_id) + assert SECRET.encode() not in plugin.transcript_bytes(invocation_id) + assert SECRET not in str(build_signed(plugin, invocation_id)) + + +def test_released_runners_keep_concurrent_invocations_isolated() -> None: + async def exercise() -> GoogleAdkTracePlugin: + plugin = GoogleAdkTracePlugin() + alpha = runner_for( + LlmAgent( + name="alpha_agent", + model=model_for_tool( + "measure_payload", arguments={"payload": "alpha-secret"} + ), + tools=[measure_payload], + ), + plugin, + "alpha_app", + ) + beta = runner_for( + LlmAgent( + name="beta_agent", + model=model_for_tool( + "measure_payload", arguments={"payload": "beta-secret"} + ), + tools=[measure_payload], + ), + plugin, + "beta_app", + ) + await asyncio.gather( + alpha.run_debug("alpha", session_id="alpha-session", quiet=True), + beta.run_debug("beta", session_id="beta-session", quiet=True), + ) + return plugin + + plugin = asyncio.run(exercise()) + assert len(plugin.invocation_ids) == 2 + for invocation_id in plugin.invocation_ids: + assert [ + (call.name, call.outcome) for call in plugin.tool_calls(invocation_id) + ] == [("measure_payload", "ok")] + + +def test_cancelled_released_run_remains_incomplete() -> None: + async def exercise() -> tuple[GoogleAdkTracePlugin, str]: + plugin = GoogleAdkTracePlugin() + model = BlockingModel() + runner = runner_for( + LlmAgent(name="blocking_agent", model=model), + plugin, + "blocking_app", + ) + task = asyncio.create_task(runner.run_debug("wait", quiet=True)) + await model.started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return plugin, plugin.invocation_ids[0] + + plugin, invocation_id = asyncio.run(exercise()) + assert b'"outcome":"incomplete"' in plugin.transcript_bytes(invocation_id) + assert plugin.tool_calls(invocation_id) == [] + + +def test_same_name_child_cannot_turn_cancelled_root_into_success() -> None: + async def exercise() -> tuple[GoogleAdkTracePlugin, str]: + plugin = GoogleAdkTracePlugin() + blocking_model = BlockingModel() + root = SequentialAgent( + name="shared_name", + sub_agents=[ + LlmAgent(name="shared_name", model=text_model()), + LlmAgent(name="blocking_child", model=blocking_model), + ], + ) + runner = InMemoryRunner( + app=App(name="same_name_app", root_agent=root, plugins=[plugin]) + ) + task = asyncio.create_task(runner.run_debug("wait", quiet=True)) + await blocking_model.started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + return plugin, plugin.invocation_ids[0] + + plugin, invocation_id = asyncio.run(exercise()) + transcript = json.loads(plugin.transcript_bytes(invocation_id)) + assert transcript["outcome"] == "incomplete" diff --git a/integrations/google-adk/test_google_adk_to_trace.py b/integrations/google-adk/test_google_adk_to_trace.py new file mode 100644 index 0000000..f7d410b --- /dev/null +++ b/integrations/google-adk/test_google_adk_to_trace.py @@ -0,0 +1,435 @@ +"""Framework-free evidence and privacy tests for the Google ADK adapter.""" + +from __future__ import annotations + +import asyncio +import pathlib +import sys +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(pathlib.Path(__file__).parent)) + +from google_adk_to_trace import ( # noqa: E402 + GoogleAdkTracePlugin, + MissingEvidence, + build_record, +) + +DIGEST = "sha256:" + "f" * 64 +SUBJECT = "spiffe://example.org/agent/google-adk" +SECRET = "customer-account-reference-that-must-not-enter-the-record" + + +def run(awaitable): + return asyncio.run(awaitable) + + +def context(invocation_id: str, function_call_id: str | None = None, agent=None): + return SimpleNamespace( + invocation_id=invocation_id, + function_call_id=function_call_id, + agent=agent, + future_payload_field=SECRET, + ) + + +def kwargs(**overrides): + values = { + "subject": SUBJECT, + "policy_bundle": b'{"rules":["no-payload-egress"]}', + "workload_digest": DIGEST, + "data_class": "confidential", + "model_provider": "test-provider", + "iat": 1_700_000_000, + } + values.update(overrides) + return values + + +def observed_plugin(invocation_id: str = "inv-1") -> GoogleAdkTracePlugin: + plugin = GoogleAdkTracePlugin() + run(plugin.before_run_callback(invocation_context=context(invocation_id))) + run( + plugin.before_model_callback( + callback_context=context(invocation_id), + llm_request=SimpleNamespace(model="test-model", contents=[SECRET]), + ) + ) + return plugin + + +def test_successful_tool_is_observed_without_payloads() -> None: + plugin = observed_plugin() + tool = SimpleNamespace(name="lookup") + ctx = context("inv-1", "call-1") + + assert ( + run( + plugin.before_tool_callback( + tool=tool, tool_args={"secret": SECRET}, tool_context=ctx + ) + ) + is None + ) + assert ( + run( + plugin.after_tool_callback( + tool=tool, + tool_args={"secret": SECRET}, + tool_context=ctx, + result={"secret": SECRET}, + ) + ) + is None + ) + run(plugin.after_run_callback(invocation_context=context("inv-1"))) + + assert [(call.name, call.outcome) for call in plugin.tool_calls("inv-1")] == [ + ("lookup", "ok") + ] + assert SECRET.encode() not in plugin.transcript_bytes("inv-1") + + +def test_tool_error_records_only_the_outcome() -> None: + plugin = observed_plugin() + tool = SimpleNamespace(name="pay") + ctx = context("inv-1", "call-1") + run( + plugin.before_tool_callback( + tool=tool, tool_args={"iban": SECRET}, tool_context=ctx + ) + ) + run( + plugin.on_tool_error_callback( + tool=tool, + tool_args={"iban": SECRET}, + tool_context=ctx, + error=RuntimeError(SECRET), + ) + ) + + assert plugin.tool_calls("inv-1")[0].outcome == "error" + assert SECRET.encode() not in plugin.transcript_bytes("inv-1") + + +def test_ambiguous_same_name_completions_are_not_paired_by_fifo() -> None: + plugin = observed_plugin() + tool = SimpleNamespace(name="parallel_tool") + first = context("inv-1") + second = context("inv-1") + run(plugin.before_tool_callback(tool=tool, tool_args={}, tool_context=first)) + run(plugin.before_tool_callback(tool=tool, tool_args={}, tool_context=second)) + run( + plugin.after_tool_callback( + tool=tool, + tool_args={}, + tool_context=second, + result={}, + ) + ) + run( + plugin.on_tool_error_callback( + tool=tool, + tool_args={}, + tool_context=first, + error=RuntimeError(SECRET), + ) + ) + + assert [call.outcome for call in plugin.tool_calls("inv-1")] == [ + "incomplete", + "incomplete", + ] + transcript = plugin.transcript_bytes("inv-1") + assert transcript.count(b'"outcome":"ok"') == 1 + assert transcript.count(b'"outcome":"error"') == 1 + + +def test_unfinished_tool_and_run_remain_incomplete() -> None: + plugin = observed_plugin() + run( + plugin.before_tool_callback( + tool=SimpleNamespace(name="slow_tool"), + tool_args={}, + tool_context=context("inv-1", "call-1"), + ) + ) + + assert plugin.tool_calls("inv-1")[0].outcome == "incomplete" + assert b'"outcome":"incomplete"' in plugin.transcript_bytes("inv-1") + + +def test_only_root_agent_completion_marks_a_run_successful() -> None: + root = SimpleNamespace(name="root") + child = SimpleNamespace(name="child") + plugin = GoogleAdkTracePlugin() + run(plugin.before_run_callback(invocation_context=context("inv-1", agent=root))) + run(plugin.after_agent_callback(agent=child, callback_context=context("inv-1"))) + run(plugin.after_run_callback(invocation_context=context("inv-1"))) + assert b'"outcome":"incomplete"' in plugin.transcript_bytes("inv-1") + + run(plugin.after_agent_callback(agent=root, callback_context=context("inv-1"))) + run(plugin.after_run_callback(invocation_context=context("inv-1"))) + assert b'"outcome":"ok"' in plugin.transcript_bytes("inv-1") + + +def test_equivalent_root_agent_instance_marks_a_run_successful() -> None: + original = SimpleNamespace(name="root") + runtime_copy = SimpleNamespace(name="root") + plugin = GoogleAdkTracePlugin() + run(plugin.before_run_callback(invocation_context=context("inv-1", agent=original))) + run( + plugin.after_agent_callback( + agent=runtime_copy, callback_context=context("inv-1") + ) + ) + run(plugin.after_run_callback(invocation_context=context("inv-1"))) + assert b'"outcome":"ok"' in plugin.transcript_bytes("inv-1") + + +def test_same_name_child_does_not_mark_the_root_complete() -> None: + root = SimpleNamespace(name="shared", parent_agent=None) + child = SimpleNamespace(name="shared", parent_agent=root) + plugin = GoogleAdkTracePlugin() + run(plugin.before_run_callback(invocation_context=context("inv-1", agent=root))) + run(plugin.after_agent_callback(agent=child, callback_context=context("inv-1"))) + run(plugin.after_run_callback(invocation_context=context("inv-1"))) + assert b'"outcome":"incomplete"' in plugin.transcript_bytes("inv-1") + + +def test_recovered_model_error_does_not_make_the_run_error() -> None: + root = SimpleNamespace(name="root") + plugin = GoogleAdkTracePlugin() + ctx = context("inv-1", agent=root) + run(plugin.before_run_callback(invocation_context=ctx)) + run( + plugin.on_model_error_callback( + callback_context=ctx, + llm_request=SimpleNamespace(model="test-model"), + error=RuntimeError(SECRET), + ) + ) + run(plugin.after_agent_callback(agent=root, callback_context=ctx)) + run(plugin.after_run_callback(invocation_context=ctx)) + assert b'"outcome":"ok"' in plugin.transcript_bytes("inv-1") + + +def test_unhandled_runner_error_marks_the_run_error() -> None: + plugin = observed_plugin() + run( + plugin.on_run_error_callback( + invocation_context=context("inv-1"), + error=RuntimeError(SECRET), + ) + ) + assert b'"outcome":"error"' in plugin.transcript_bytes("inv-1") + + +def test_completion_without_start_is_not_dropped() -> None: + plugin = observed_plugin() + run( + plugin.after_tool_callback( + tool=SimpleNamespace(name="late_tool"), + tool_args={}, + tool_context=context("inv-1", "call-9"), + result={}, + ) + ) + + call = plugin.tool_calls("inv-1")[0] + assert (call.name, call.outcome, call.observed_start) == ("late_tool", "ok", False) + + +def test_missing_tool_name_is_retained_as_unavailable() -> None: + plugin = observed_plugin() + ctx = context("inv-1", "call-1") + tool = SimpleNamespace(name="") + run(plugin.before_tool_callback(tool=tool, tool_args={}, tool_context=ctx)) + run( + plugin.after_tool_callback(tool=tool, tool_args={}, tool_context=ctx, result={}) + ) + assert plugin.tool_calls("inv-1")[0].name is None + assert b'"tool":null' in plugin.transcript_bytes("inv-1") + + +def test_model_payload_and_unknown_fields_are_ignored() -> None: + plugin = observed_plugin() + tool = SimpleNamespace(name="lookup", future_secret=SECRET) + ctx = context("inv-1", "call-1") + run( + plugin.before_tool_callback( + tool=tool, tool_args={"secret": SECRET}, tool_context=ctx + ) + ) + + record = plugin.build_record("inv-1", **kwargs()) + assert SECRET not in str(record) + assert SECRET.encode() not in plugin.transcript_bytes("inv-1") + + +def test_transcript_is_deterministic_and_order_sensitive() -> None: + first = observed_plugin() + second = observed_plugin() + for plugin, names in ((first, ["a", "b"]), (second, ["b", "a"])): + for index, name in enumerate(names): + ctx = context("inv-1", f"call-{index}") + tool = SimpleNamespace(name=name) + run(plugin.before_tool_callback(tool=tool, tool_args={}, tool_context=ctx)) + run( + plugin.after_tool_callback( + tool=tool, tool_args={}, tool_context=ctx, result={} + ) + ) + + assert first.transcript_bytes("inv-1") == first.transcript_bytes("inv-1") + assert first.transcript_bytes("inv-1") != second.transcript_bytes("inv-1") + + +def test_concurrent_invocation_state_does_not_cross_contaminate() -> None: + async def exercise() -> GoogleAdkTracePlugin: + plugin = GoogleAdkTracePlugin() + + async def one(invocation_id: str, tool_name: str) -> None: + ctx = context(invocation_id, "call-1") + tool = SimpleNamespace(name=tool_name) + await plugin.before_run_callback(invocation_context=ctx) + await plugin.before_model_callback( + callback_context=ctx, + llm_request=SimpleNamespace(model=f"model-{invocation_id}"), + ) + await plugin.before_tool_callback(tool=tool, tool_args={}, tool_context=ctx) + await asyncio.sleep(0) + await plugin.after_tool_callback( + tool=tool, + tool_args={}, + tool_context=ctx, + result={}, + ) + await plugin.after_run_callback(invocation_context=ctx) + + await asyncio.gather(one("inv-a", "alpha"), one("inv-b", "beta")) + return plugin + + plugin = asyncio.run(exercise()) + assert [call.name for call in plugin.tool_calls("inv-a")] == ["alpha"] + assert [call.name for call in plugin.tool_calls("inv-b")] == ["beta"] + + +def test_multiple_observed_models_require_an_explicit_choice() -> None: + plugin = observed_plugin() + run( + plugin.before_model_callback( + callback_context=context("inv-1"), + llm_request=SimpleNamespace(model="second-model"), + ) + ) + + with pytest.raises(MissingEvidence, match="multiple model ids"): + plugin.build_record("inv-1", **kwargs()) + with pytest.raises(MissingEvidence, match="multiple model ids"): + plugin.build_record("inv-1", **kwargs(model_id="router-selection")) + + +def test_observed_model_id_cannot_be_relabelled() -> None: + plugin = observed_plugin() + with pytest.raises(MissingEvidence, match="conflicts with observed model"): + plugin.build_record("inv-1", **kwargs(model_id="different-model")) + + +def test_provider_is_never_guessed_from_the_model_name() -> None: + plugin = observed_plugin() + with pytest.raises(MissingEvidence, match="model_provider and model_id"): + plugin.build_record("inv-1", **kwargs(model_provider=None)) + + +def test_record_validates_after_signing() -> None: + TrustRecord = pytest.importorskip("agentrust_trace.models").TrustRecord + sign = pytest.importorskip("agentrust_trace.sign") + plugin = observed_plugin() + record = sign.sign_record( + plugin.build_record("inv-1", **kwargs()), sign.generate_key() + ) + + parsed = TrustRecord.model_validate(record) + assert parsed.runtime.platform == "software-only" + assert parsed.appraisal.status == "none" + assert "origin" not in record + assert "transparency" not in record + assert "tool_transcript" not in record + + +def test_discard_removes_retained_invocation() -> None: + plugin = observed_plugin() + assert plugin.discard("inv-1") is True + assert plugin.discard("inv-1") is False + with pytest.raises(MissingEvidence, match="no evidence retained"): + plugin.transcript_bytes("inv-1") + + +@pytest.mark.parametrize("subject", ["agent", "spiffe://missing-path"]) +def test_invalid_subject_is_refused(subject: str) -> None: + with pytest.raises(MissingEvidence, match="may not invent identity"): + build_record( + **kwargs(subject=subject, model_id="test-model"), + transcript=b"{}", + tool_count=0, + ) + + +def test_empty_policy_is_refused() -> None: + with pytest.raises(MissingEvidence, match="policy bytes"): + build_record( + **kwargs(policy_bundle=b"", model_id="test-model"), + transcript=b"{}", + tool_count=0, + ) + + +def test_invalid_workload_digest_is_refused() -> None: + with pytest.raises(MissingEvidence, match="artifact digest"): + build_record( + **kwargs(workload_digest="sha256:placeholder", model_id="test-model"), + transcript=b"{}", + tool_count=0, + ) + + +def test_invalid_enforcement_mode_is_refused() -> None: + with pytest.raises(MissingEvidence, match="enforcement_mode"): + build_record( + **kwargs(enforcement_mode="monitor", model_id="test-model"), + transcript=b"{}", + tool_count=0, + ) + + +def test_software_only_attestation_is_refused() -> None: + with pytest.raises(MissingEvidence, match="omit attestation"): + build_record( + **kwargs(model_id="test-model"), + transcript=b"{}", + tool_count=0, + attestation={"platform": "software-only", "measurement": DIGEST}, + ) + + +def test_attestation_measurement_must_be_a_digest() -> None: + with pytest.raises(MissingEvidence, match="measured digest"): + build_record( + **kwargs(model_id="test-model"), + transcript=b"{}", + tool_count=0, + attestation={"platform": "intel-tdx", "measurement": "unknown"}, + ) + + +def test_attestation_lifts_the_same_record_to_hardware() -> None: + record = build_record( + **kwargs(model_id="test-model"), + transcript=b"{}", + tool_count=0, + attestation={"platform": "intel-tdx", "measurement": DIGEST}, + ) + assert record["runtime"] == {"platform": "intel-tdx", "measurement": DIGEST} diff --git a/marketplace/catalog.json b/marketplace/catalog.json index 72f6db3..7f6f97d 100644 --- a/marketplace/catalog.json +++ b/marketplace/catalog.json @@ -1,6 +1,6 @@ { "catalog_version": 1, - "count": 16, + "count": 17, "integrations": [ { "name": "Claude Code", @@ -246,6 +246,28 @@ "policy" ] }, + { + "name": "Google ADK", + "package_name": "Google ADK", + "vendor": "agentrust-io", + "description": "Emits a TRACE Trust Record from Google ADK callbacks using callback-visible tool and model identity without payloads.", + "path": "integrations/google-adk", + "url": "https://github.com/agentrust-io/integrations/tree/main/integrations/google-adk", + "homepage": null, + "repository": "https://github.com/agentrust-io/integrations", + "tier": "community", + "stack": [ + "TRACE" + ], + "category": "Frameworks", + "mark": "ADK", + "featured": null, + "keywords": [ + "callback", + "framework", + "tools" + ] + }, { "name": "Nobulex", "package_name": "Nobulex", diff --git a/noxfile.py b/noxfile.py index b75ae21..f307757 100644 --- a/noxfile.py +++ b/noxfile.py @@ -6,7 +6,7 @@ nox.options.default_venv_backend = "venv" -nox.options.sessions = ["capture_core", "trace_adapters", "capture_engines", "framework_adapters", "shadow_ai"] +nox.options.sessions = ["capture_core", "trace_adapters", "capture_engines", "framework_adapters", "google_adk_adapter", "shadow_ai"] def pytest(session: nox.Session, *paths: str) -> None: @@ -47,6 +47,18 @@ def framework_adapters(session: nox.Session) -> None: pytest(session, "integrations/langchain/test_langgraph_interop.py") +@nox.session(python="3.12") +def google_adk_adapter(session: nox.Session) -> None: + session.install("agentrust-trace==0.9.0", "pytest>=8") + pytest(session, "integrations/google-adk/test_google_adk_to_trace.py") + session.install("google-adk==2.7.1", "agentrust-trace-tests==0.5.0") + pytest( + session, + "integrations/google-adk/test_google_adk_to_trace.py", + "integrations/google-adk/test_google_adk_interop.py", + ) + + @nox.session(python="3.12") def shadow_ai(session: nox.Session) -> None: session.install("pytest>=8", "pyyaml")