From 67168d3311e3e12adf9a063200acacdbe0bf08aa Mon Sep 17 00:00:00 2001 From: Dawn Jeong Date: Wed, 23 Sep 2026 16:13:17 -0700 Subject: [PATCH 1/6] flightcheck: add canonical DA connection-reference reader + contract test (AB#7852506, AB#7852495, AB#7852511) Extract the shared minimalBots components connection-reference reader into checks/_da_connection_refs.py as the single source that DV-CONN-001 (active agent), ENV-004 (environment-wide, de-duped by logical name) and the Workday shared-parameter checks (WD-ENV-001/WD-REST-001) all read through, so the per-check re-point PRs stop adding divergent copies of it. The module is the superset of PR #317's reader plus a public read_all_agents_connection_references() that de-dupes by connection-reference logical name for ENV-004. Brings the validated/documented mock builders the checks need and a direct pure-logic contract test (18 cases) covering normalization, None-gating (SKIP), fail-loudly ValueError, JSON-string sharedConnectionParameters parsing, and the Workday shared-parameter sweep. Foundation for the 6 DA re-point PRs to rebase onto (single shared reader, no add/add collision). Full flightcheck suite: 1142 passed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: e25cf992-09c9-49b8-9971-2a19cb2fcb05 --- .../flightcheck/checks/_da_connection_refs.py | 245 ++++++++++++++++++ .../checks/test_da_connection_refs.py | 205 +++++++++++++++ tests/mocks/agentbuilder_connectivity.py | 112 ++++++++ 3 files changed, 562 insertions(+) create mode 100644 solutions/ess-maker-skills/scripts/flightcheck/checks/_da_connection_refs.py create mode 100644 tests/flightcheck/checks/test_da_connection_refs.py diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/_da_connection_refs.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/_da_connection_refs.py new file mode 100644 index 000000000..3e40167da --- /dev/null +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/_da_connection_refs.py @@ -0,0 +1,245 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Canonical reader for Declarative Agent connection references (minimalBots +components API), shared so the DA connection checks cannot drift apart. + +Consumers: + * ``DV-CONN-001`` (checks/workday_extension.py) -> the single active agent's + Workday SOAP reference, via ``read_active_agent_connection_references``. + * ``ENV-004`` (checks/environment.py) -> every configured agent's references, + environment-wide and de-duped by logical name, via + ``read_all_agents_connection_references``. + * The Workday shared-parameter checks (checks/workday.py) -> per-agent Workday + ``sharedConnectionParameters`` via ``workday_shared_connection_parameters``. + +Read shape: ``POST .../components`` -> ``connectionReferenceChanges`` (cassette +``agentbuilder_readiness.yaml``, the same endpoint + shape the shipped native +``DA-CONN-001`` check consumes). + +Fail-loudly contract: + * a missing ``connectionReferenceChanges`` key means genuine absence -> ``[]``; + * a present-but-malformed shape raises ``ValueError`` so the owning check + degrades to a WARNING rather than reporting a confident but wrong verdict; + * a read that cannot be attempted at all (no AgentBuilder client, or no + configured agent botId) returns ``None`` so the caller SKIPs. +""" + +from __future__ import annotations + +import json +from typing import Any + + +WORKDAY_SOAP_CONNECTOR_SUFFIX = "/apis/shared_workdaysoap" + + +def agent_bot_ids(config: dict[str, Any]) -> list[str]: + """Return configured bot IDs from multi-agent and single-agent config.""" + bot_ids: list[str] = [] + for agent in config.get("agents", []) or []: + bid = (agent or {}).get("botId") + if isinstance(bid, str) and bid.strip(): + bot_ids.append(bid.strip()) + single = (config.get("agent") or {}).get("botId") + if isinstance(single, str) and single.strip(): + bot_ids.append(single.strip()) + + seen: set[str] = set() + ordered: list[str] = [] + for bot_id in bot_ids: + folded = bot_id.casefold() + if folded not in seen: + seen.add(folded) + ordered.append(bot_id) + return ordered + + +def _bot_connection_references(client, bot_id: str) -> list[dict[str, Any]]: + """Fetch + normalize one agent's connection references from the minimalBots + components API. + + Raises ``ValueError`` for a malformed ``connectionReferenceChanges`` shape + so the owning check reports a WARNING instead of overclaiming. + """ + changeset = client.fetch_components(bot_id) or {} + changes = changeset.get("connectionReferenceChanges") + if changes is None: + return [] + if not isinstance(changes, list): + raise ValueError( + "Component fetch returned invalid connectionReferenceChanges." + ) + + refs: list[dict[str, Any]] = [] + for change in changes: + item = ( + change.get("connectionReference") + if isinstance(change, dict) + else None + ) + if not isinstance(item, dict): + continue + refs.append( + { + "botid": bot_id, + "connectionreferencelogicalname": item.get( + "connectionReferenceLogicalName" + ), + "connectorid": item.get("connectorId"), + "connectionid": item.get("connectionId"), + "sharedconnectionparameters": item.get( + "sharedConnectionParameters" + ), + } + ) + return refs + + +def read_active_agent_connection_references(runner) -> list[dict[str, Any]] | None: + """The single active agent's DA connection references (config + ``agent.botId``), or ``None`` when the AgentBuilder client or the + active-agent botId is unavailable. + + Used by ``DV-CONN-001`` (checks/workday_extension.py), which validates the + Workday SOAP connection reference on the agent under check. Scoping this to + the active agent — not every configured agent — keeps the check from + reporting on a Workday reference that belongs to a different agent. + Raises ``ValueError`` for malformed components payloads. + """ + client = getattr(runner, "agentbuilder", None) + config = getattr(runner, "config", None) or {} + agent_id = (config.get("agent") or {}).get("botId") + if client is None or not agent_id: + return None + return _bot_connection_references(client, agent_id) + + +def _all_agents_connection_references(runner) -> list[dict[str, Any]] | None: + """Every configured agent's DA connection references (multi-agent and + single-agent config), preserving per-agent rows, or ``None`` when the + AgentBuilder client is unavailable or no agent botId is configured. + + Used by the Workday shared-parameter sweep, which must inspect each + configured agent's own Workday reference rather than only the active one. + Raises ``ValueError`` for malformed components payloads. + """ + client = getattr(runner, "agentbuilder", None) + config = getattr(runner, "config", None) or {} + bot_ids = agent_bot_ids(config) + if client is None or not bot_ids: + return None + + refs: list[dict[str, Any]] = [] + for bot_id in bot_ids: + refs.extend(_bot_connection_references(client, bot_id)) + return refs + + +def read_all_agents_connection_references( + runner, +) -> list[dict[str, Any]] | None: + """Every configured agent's references, de-duped by connection-reference + logical name (first occurrence wins, order preserved), or ``None`` when the + AgentBuilder client is unavailable or no agent botId is configured. + + Used by ``ENV-004`` (checks/environment.py), which is environment-wide + across every agent under check and reports one row per distinct logical + name. The per-agent (non-de-duped) view is ``_all_agents_connection_references``, + which the Workday shared-parameter sweep uses instead. + """ + refs = _all_agents_connection_references(runner) + if refs is None: + return None + seen: set[str] = set() + deduped: list[dict[str, Any]] = [] + for ref in refs: + key = (ref.get("connectionreferencelogicalname") or "").casefold() + if key and key in seen: + continue + if key: + seen.add(key) + deduped.append(ref) + return deduped + + +def _shared_parameter_value(raw_value: Any) -> str: + if isinstance(raw_value, dict): + raw_value = raw_value.get("value") + if raw_value is None: + return "" + return str(raw_value).strip() + + +def shared_connection_parameter_values(ref: dict[str, Any]) -> dict[str, str]: + """Return ``sharedConnectionParameters.values`` as a string map. + + A present-but-malformed shape raises ``ValueError`` because the components + payload no longer matches the validated contract. + """ + params = ref.get("sharedconnectionparameters") + if params is None: + return {} + # Live AgentBuilder returns sharedConnectionParameters as a JSON string, + # not a nested object (observed on a live connection reference), so parse + # the string before validating the shape. + if isinstance(params, str): + text = params.strip() + if not text: + return {} + try: + params = json.loads(text) + except ValueError as exc: + raise ValueError( + "Component fetch returned invalid sharedConnectionParameters." + ) from exc + if not isinstance(params, dict): + raise ValueError( + "Component fetch returned invalid sharedConnectionParameters." + ) + raw_values = params.get("values") + if raw_values is None: + return {} + if not isinstance(raw_values, dict): + raise ValueError( + "Component fetch returned invalid sharedConnectionParameters.values." + ) + return { + str(key): value + for key, raw_value in raw_values.items() + if isinstance(key, str) + if (value := _shared_parameter_value(raw_value)) + } + + +def workday_shared_connection_parameters( + runner, +) -> tuple[dict[str, str] | None, str]: + """Return Workday ``sharedConnectionParameters.values`` from components. + + ``values is None`` means the check could not run because AgentBuilder or a + botId is unavailable. ``values == {}`` means the check ran and observed a + missing Workday reference or missing shared parameters. + """ + refs = _all_agents_connection_references(runner) + if refs is None: + return None, ( + "AgentBuilder client or a configured agent botId not available" + ) + + found_workday_ref = False + for ref in refs: + connector_id = str(ref.get("connectorid") or "").casefold().rstrip("/") + if connector_id.endswith(WORKDAY_SOAP_CONNECTOR_SUFFIX): + found_workday_ref = True + values = shared_connection_parameter_values(ref) + if values: + return values, "" + + if found_workday_ref: + return {}, ( + "Workday connection reference is missing " + "sharedConnectionParameters.values" + ) + + return {}, "Workday connection reference was not found" diff --git a/tests/flightcheck/checks/test_da_connection_refs.py b/tests/flightcheck/checks/test_da_connection_refs.py new file mode 100644 index 000000000..e3b4c42e7 --- /dev/null +++ b/tests/flightcheck/checks/test_da_connection_refs.py @@ -0,0 +1,205 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Contract tests for the canonical Declarative Agent connection-reference +reader (``checks/_da_connection_refs.py``), the single shared module that +``DV-CONN-001`` (active agent), ``ENV-004`` (environment-wide, de-duped) and the +Workday shared-parameter checks all read through. + +These are pure-logic tests: a duck-typed fake AgentBuilder client returns +minimalBots components payloads built from +``tests.mocks.agentbuilder_connectivity`` (``MOCK_STATUS == "validated"``), so +no network replay or cassette is needed (same inline-fake approach as +``test_agent_handoff.py``). Every component shape traces to the validated +``components()`` builder or its documented ``sharedConnectionParameters`` +variant; none is invented here. +""" + +from __future__ import annotations + +from typing import Any + +import pytest + +from flightcheck.checks import _da_connection_refs as reader +from tests.mocks import agentbuilder_connectivity as ab + + +class _FakeClient: + """AgentBuilder stand-in: ``fetch_components(bot_id)`` returns the payload + registered for that bot id (empty ``{}`` when none is registered).""" + + def __init__(self, payload_by_bot: dict[str, dict[str, Any]]): + self._payload_by_bot = payload_by_bot + + def fetch_components(self, bot_id: str) -> dict[str, Any]: + return self._payload_by_bot.get(bot_id, {}) + + +class _FakeRunner: + def __init__(self, client: _FakeClient | None, config: dict[str, Any]): + self.agentbuilder = client + self.config = config + + +# -------------------------------------------------------------------------- +# agent_bot_ids +# -------------------------------------------------------------------------- + +def test_agent_bot_ids_unions_multi_and_single_and_dedups_casefold(): + config = { + "agents": [{"botId": "Agent-A"}, {"botId": "Agent-B"}, {"botId": "agent-a"}], + "agent": {"botId": "Agent-B"}, + } + # agent-a folds onto Agent-A (dup); single Agent-B folds onto Agent-B (dup). + assert reader.agent_bot_ids(config) == ["Agent-A", "Agent-B"] + + +def test_agent_bot_ids_single_only(): + assert reader.agent_bot_ids({"agent": {"botId": "SOLO"}}) == ["SOLO"] + + +def test_agent_bot_ids_ignores_blank_and_non_string(): + config = {"agents": [{"botId": ""}, {"botId": None}, {"botId": " X "}]} + assert reader.agent_bot_ids(config) == ["X"] + + +# -------------------------------------------------------------------------- +# read_active_agent_connection_references (DV-CONN-001 surface) +# -------------------------------------------------------------------------- + +def test_read_active_none_when_no_client(): + runner = _FakeRunner(None, {"agent": {"botId": "BOT"}}) + assert reader.read_active_agent_connection_references(runner) is None + + +def test_read_active_none_when_no_active_bot_id(): + # Only multi-agent config; no config["agent"].botId -> active read SKIPs. + runner = _FakeRunner(_FakeClient({}), {"agents": [{"botId": "BOT"}]}) + assert reader.read_active_agent_connection_references(runner) is None + + +def test_read_active_normalizes_workday_row(): + payload = ab.components_with_references( + references=[ab.workday_connection_reference(connection_id="wd-conn-1")] + ) + runner = _FakeRunner(_FakeClient({"BOT": payload}), {"agent": {"botId": "BOT"}}) + rows = reader.read_active_agent_connection_references(runner) + assert len(rows) == 1 + row = rows[0] + assert row["botid"] == "BOT" + assert row["connectorid"].endswith("/apis/shared_workdaysoap") + assert row["connectionid"] == "wd-conn-1" + + +def test_read_active_missing_change_set_is_empty_not_none(): + runner = _FakeRunner(_FakeClient({"BOT": {}}), {"agent": {"botId": "BOT"}}) + assert reader.read_active_agent_connection_references(runner) == [] + + +def test_read_active_malformed_change_set_raises(): + runner = _FakeRunner( + _FakeClient({"BOT": {"connectionReferenceChanges": "not-a-list"}}), + {"agent": {"botId": "BOT"}}, + ) + with pytest.raises(ValueError): + reader.read_active_agent_connection_references(runner) + + +# -------------------------------------------------------------------------- +# read_all_agents_connection_references (ENV-004 surface: env-wide, de-duped) +# -------------------------------------------------------------------------- + +def test_read_all_none_when_no_client(): + runner = _FakeRunner(None, {"agents": [{"botId": "A"}]}) + assert reader.read_all_agents_connection_references(runner) is None + + +def test_read_all_dedups_by_logical_name_first_wins(): + ref_a = ab.workday_connection_reference( + connection_id="c1", logical_name="ns.shared_workdaysoap" + ) + ref_b = ab.workday_connection_reference( + connection_id="c2", logical_name="ns.shared_workdaysoap" + ) + client = _FakeClient( + { + "A": ab.components_with_references(references=[ref_a]), + "B": ab.components_with_references(references=[ref_b]), + } + ) + runner = _FakeRunner(client, {"agents": [{"botId": "A"}, {"botId": "B"}]}) + rows = reader.read_all_agents_connection_references(runner) + assert len(rows) == 1 + assert rows[0]["connectionid"] == "c1" + + +# -------------------------------------------------------------------------- +# shared_connection_parameter_values +# -------------------------------------------------------------------------- + +def test_scp_values_parsed_from_json_string(): + ref = { + "sharedconnectionparameters": ab.shared_connection_parameters_json_string( + rest_base_uri="https://wd.example.com/ccx/api" + ) + } + values = reader.shared_connection_parameter_values(ref) + assert values["restBaseUri"] == "https://wd.example.com/ccx/api" + + +def test_scp_values_parsed_from_nested_object(): + ref = {"sharedconnectionparameters": ab.shared_connection_parameters(tenant_name="mocktenant")} + values = reader.shared_connection_parameter_values(ref) + assert values["tenantName"] == "mocktenant" + + +def test_scp_values_missing_is_empty_map(): + assert reader.shared_connection_parameter_values({}) == {} + + +def test_scp_values_malformed_json_string_raises(): + ref = {"sharedconnectionparameters": "{not valid json"} + with pytest.raises(ValueError): + reader.shared_connection_parameter_values(ref) + + +# -------------------------------------------------------------------------- +# workday_shared_connection_parameters (WD-ENV-001 / WD-REST-001 surface) +# -------------------------------------------------------------------------- + +def test_wscp_found_with_values(): + ref = ab.workday_connection_reference( + shared_connection_parameters=ab.shared_connection_parameters_json_string() + ) + payload = ab.components_with_references(references=[ref]) + runner = _FakeRunner(_FakeClient({"BOT": payload}), {"agent": {"botId": "BOT"}}) + values, message = reader.workday_shared_connection_parameters(runner) + assert message == "" + assert values["tenantName"] == "mocktenant" + + +def test_wscp_found_but_missing_values(): + ref = ab.workday_connection_reference(shared_connection_parameters=None) + payload = ab.components_with_references(references=[ref]) + runner = _FakeRunner(_FakeClient({"BOT": payload}), {"agent": {"botId": "BOT"}}) + values, message = reader.workday_shared_connection_parameters(runner) + assert values == {} + assert "missing" in message.lower() + + +def test_wscp_workday_reference_not_found(): + # components() carries only the ServiceNow reference, no Workday one. + runner = _FakeRunner( + _FakeClient({"BOT": ab.components()}), {"agent": {"botId": "BOT"}} + ) + values, message = reader.workday_shared_connection_parameters(runner) + assert values == {} + assert "not found" in message.lower() + + +def test_wscp_none_when_client_unavailable(): + runner = _FakeRunner(None, {"agent": {"botId": "BOT"}}) + values, message = reader.workday_shared_connection_parameters(runner) + assert values is None + assert message diff --git a/tests/mocks/agentbuilder_connectivity.py b/tests/mocks/agentbuilder_connectivity.py index 9fb6c123d..f666f928c 100644 --- a/tests/mocks/agentbuilder_connectivity.py +++ b/tests/mocks/agentbuilder_connectivity.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json from typing import Any, Iterable import responses @@ -20,6 +21,7 @@ MOCK_AGENT_ID = "00000000-0000-0000-0000-000000002222" MOCK_FAMILY_ID = "00000000-0000-0000-0000-000000003333" MOCK_CONNECTION_ID = "mock-servicenow-connection" +MOCK_WORKDAY_CONNECTION_ID = "mock-workday-connection" MOCK_AGENTBUILDER_BASE = ( "https://00000000000000000000000000000000." "0.environment.api.test.powerplatform.com" @@ -76,6 +78,116 @@ def components() -> dict[str, Any]: } +def connection_reference_change( + *, + connector: str, + connection_id: str | None, + logical_name: str | None = None, + shared_connection_parameters: dict[str, Any] | None = None, +) -> dict[str, Any]: + """One ``connectionReferenceChanges`` entry in the validated minimalBots + components shape (cassette ``agentbuilder_readiness.yaml``; the same shape + the shipped native ``DA-CONN-001`` check consumes). Only the ``connectorId`` + value varies from the captured ServiceNow reference, so this is + same-endpoint value variance and needs no new cassette (see + ``scripts/flightcheck/AGENTS.md``). ``connection_id=None`` models an unbound + reference. + """ + reference = { + "connectionReferenceLogicalName": ( + logical_name + or f"gptagent_mockemployeeselfservice.{connector}" + ), + "connectorId": ( + f"/providers/Microsoft.PowerApps/apis/{connector}" + ), + "connectionId": connection_id, + } + if shared_connection_parameters is not None: + reference["sharedConnectionParameters"] = ( + shared_connection_parameters + ) + return { + "changeType": "Insert", + "connectionReference": reference, + } + + +def workday_connection_reference( + *, + connection_id: str | None = MOCK_WORKDAY_CONNECTION_ID, + logical_name: str | None = None, + shared_connection_parameters: dict[str, Any] | None = None, +) -> dict[str, Any]: + """The Workday SOAP (``shared_workdaysoap``) connection-reference variant + that ``DV-CONN-001`` filters on.""" + return connection_reference_change( + connector="shared_workdaysoap", + connection_id=connection_id, + logical_name=logical_name, + shared_connection_parameters=shared_connection_parameters, + ) + + +def shared_connection_parameters( + *, + rest_base_uri: str | None = "https://wd.example.com/ccx/api", + tenant_name: str | None = "mocktenant", + resource_uri: str | None = "https://wd.example.com", + token_uri: str | None = "https://wd.example.com/ccx/oauth2/mocktenant/token", + client_id: str | None = "mock-client-id", +) -> dict[str, Any]: + """Workday ``sharedConnectionParameters`` from documented sources. + + Source (documented): + ``tools/ess-ca-to-da/reference/hr/agent.yml`` captures a real + ServiceNow ``sharedConnectionParameters`` entry using the nested + ``values..value`` wrapper shape. The Workday-specific fields mirror + the public Workday connector definition documented at + ``https://learn.microsoft.com/connectors/workdaysoap/``: + ``restBaseUri``, ``tenantName``, ``token:ResourceUri``, + ``token:WorkdayTokenUri``, and ``token:WorkdayClientId``. These + Workday-specific keys are not yet captured from a live AgentBuilder + components response. + """ + values: dict[str, dict[str, str]] = {} + for key, value in ( + ("restBaseUri", rest_base_uri), + ("tenantName", tenant_name), + ("token:ResourceUri", resource_uri), + ("token:WorkdayTokenUri", token_uri), + ("token:WorkdayClientId", client_id), + ): + if value is not None: + values[key] = {"value": value} + return {"values": values} + + +def shared_connection_parameters_json_string(**kwargs: Any) -> str: + """``sharedConnectionParameters`` as the JSON string the live AgentBuilder + components response returns, rather than a nested object. + + Source (documented): a live ServiceNow connection reference encodes + ``sharedConnectionParameters`` as a JSON string, so checks must parse it + before reading ``values``. + """ + return json.dumps(shared_connection_parameters(**kwargs)) + + +def components_with_references( + *, + references: Iterable[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """``components()`` with its ``connectionReferenceChanges`` replaced by the + given references (``None`` -> an empty list, modelling an agent with no + connection references).""" + payload = components() + payload["connectionReferenceChanges"] = ( + [] if references is None else list(references) + ) + return payload + + def connection( *, connection_id: str = MOCK_CONNECTION_ID, From dba6714282a38a20ab2cb831600a4c44edc26cc4 Mon Sep 17 00:00:00 2001 From: Dawn Jeong Date: Tue, 22 Sep 2026 14:50:24 -0700 Subject: [PATCH 2/6] flightcheck: re-point DV-CONN-001 to Declarative Agent components API (7852506) Read the Workday SOAP connection reference from the Declarative Agent minimalBots components API (runner.agentbuilder.fetch_components) instead of the Custom Agent Dataverse connectionreferences query, so DV-CONN-001 works against DA-GA agents that no longer expose Dataverse. Verdict simplifies to found + bound (connectionId present); the Dataverse-only statuscode and multiple-ref branches are dropped since the components shape carries neither. BAP owner echo is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../flightcheck/checks/workday_extension.py | 195 +++++++---------- .../scripts/flightcheck/registry.py | 9 +- .../checks/test_workday_extension.py | 197 ++++++------------ tests/flightcheck/test_registry.py | 12 +- 4 files changed, 149 insertions(+), 264 deletions(-) diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py index 9f023bc01..0dc741a17 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py @@ -20,11 +20,12 @@ (never assert a verdict from an unconfirmed API response shape) — this checkpoint echoes the observed ``connectionParametersSet.name`` for the operator to confirm, rather than PASS/FAIL on a guessed value. - * ``DV-CONN-001`` (S5.4) — the Dataverse connection reference the extension - pack ships (``…_92b66``, connector ``shared_commondataserviceforapps``) is - bound to an **active** connection, and its owner is echoed so the operator - can confirm it is their **own** account. Programmatic PASS/FAIL on a - documented-tier Dataverse ``connectionreferences`` read. + * ``DV-CONN-001`` (S5.4) — the Workday SOAP connection reference reported by + the Declarative Agent minimalBots components API is bound (``connectionId`` + present), and its owner is echoed so the operator can confirm it is their + **own** account. Programmatic PASS/FAIL on the validated minimalBots + components read (same endpoint + ``connectionReferenceChanges`` shape as the + shipped native ``DA-CONN-001`` check). * ``WD-REST-001`` (S5.5) — the captured ``restBaseUrl`` is present and **trimmed to** ``/api``. Pure-config check, no client. * ``WD-REST-002`` (S5.7) — the agent's ``user-context-setup.mcs.yml`` topic @@ -43,7 +44,7 @@ whole run. * **One CheckResult per checkpoint** (principle 7). * **No guessed API shapes** — the two API-backed checks read documented fields - only (Dataverse ``connectionid`` / ``statuscode``; BAP + only (minimalBots ``connectionReferenceChanges`` connector/connection ids; BAP ``connectionParametersSet.name`` / ``createdBy``), and degrade gracefully when a client is unavailable. * **Every** ``CheckResult`` declares ``roles=`` (enforced by @@ -52,18 +53,11 @@ from __future__ import annotations -import os import re -import sys from pathlib import Path from ..runner import CheckResult, Priority, Role, Status -# scripts/auth.py is on sys.path via cli.py at runtime (tests add it too); this -# mirrors checks/environment.py's top-level import so query_all is patchable as -# flightcheck.checks.workday_extension.query_all. -from auth import query_all # noqa: E402 - DOC_BASE = ( "https://learn.microsoft.com/en-us/copilot/microsoft-365/" "employee-self-service" @@ -91,12 +85,9 @@ _WORKDAY_RUNTIME_REF_LOGICAL_NAME = ( "msdyn_sharedworkdaysoap_workdayruntime" ) -# The Dataverse connection reference the simplified pack ships. -_DATAVERSE_CONNECTOR_SUFFIX = "/apis/shared_commondataserviceforapps" -_DATAVERSE_REF_SUFFIX = "92b66" -_DATAVERSE_RUNTIME_REF_LOGICAL_NAME = ( - "msdyn_sharedcommondataserviceforapps_workdayruntime" -) +# The Workday SOAP connection reference the Declarative Agent reports via the +# minimalBots components API (connector ``shared_workdaysoap``). +_WORKDAY_CONNECTOR_SUFFIX = "/apis/shared_workdaysoap" _REF_SUFFIX_RE = re.compile(r"_([0-9a-f]{5})$") # ---- Local user-context topic (WD-REST-002) ---- @@ -108,7 +99,7 @@ "Workday connection authentication type is Microsoft Entra ID Integrated" ) _DV_CONN_DESC = ( - "Dataverse connection reference bound to an active connection you own" + "Workday SOAP connection reference bound to a connection you own" ) _REST_URL_DESC = "Workday REST base URL present and trimmed to '/api'" _REDIRECT_DESC = ( @@ -155,14 +146,6 @@ def _is_workday_auth_ref(logical_name) -> bool: ) -def _is_dataverse_runtime_ref(logical_name) -> bool: - normalized = str(logical_name or "").casefold() - return ( - _ref_suffix(logical_name) == _DATAVERSE_REF_SUFFIX - or normalized == _DATAVERSE_RUNTIME_REF_LOGICAL_NAME.casefold() - ) - - def _host_of(url: str) -> str: """Return the host portion of an ``https://host/…`` URL for display.""" match = re.match(r"https?://([^/]+)", str(url).strip()) @@ -192,26 +175,40 @@ def _resolve_owner(props: dict) -> str: def _query_connection_references(runner): - """Return all Dataverse ``connectionreferences`` rows, or ``None`` when the - Dataverse token/endpoint is not available. - - Documented-tier read (Dataverse Web API v9.2) — no cassette required; tests - stub ``query_all``. + """Return the agent's connection references from the Declarative Agent + minimalBots components API, normalized to the row shape + ``_check_dv_connection`` consumes, or ``None`` when the AgentBuilder client + or the active-agent ``botId`` is unavailable. + + Validated-tier read (minimalBots ``POST …/components``). The same endpoint + and ``connectionReferenceChanges`` shape already back the shipped native + ``DA-CONN-001`` check (``checks/native_agent.py``); see + ``tests/fixtures/cassettes/INDEX.md`` and ``tests/mocks/ + agentbuilder_connectivity.py``. Any ``AgentBuilderHTTPError`` propagates so + the dispatcher degrades this checkpoint to a WARNING (fail loudly). """ - env_url = getattr(runner, "env_url", None) - dv_token = getattr(runner, "dv_token", None) - if not env_url or not dv_token: + client = getattr(runner, "agentbuilder", None) + config = getattr(runner, "config", None) or {} + agent_id = (config.get("agent") or {}).get("botId") + if client is None or not agent_id: return None - # Belt-and-suspenders: keep scripts/ importable even if the module was - # imported before cli.py put it on the path. - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - return query_all( - env_url, - dv_token, - "connectionreferences", - "connectionreferenceid,connectionreferencelogicalname," - "connectionreferencedisplayname,connectorid,connectionid,statuscode", - ) + changeset = client.fetch_components(agent_id) or {} + changes = changeset.get("connectionReferenceChanges") + if not isinstance(changes, list): + return [] + refs = [] + for change in changes: + ref = (change or {}).get("connectionReference") or {} + refs.append( + { + "connectionreferencelogicalname": ref.get( + "connectionReferenceLogicalName" + ), + "connectorid": ref.get("connectorId"), + "connectionid": ref.get("connectionId"), + } + ) + return refs def _get_connections(runner): @@ -366,7 +363,7 @@ def _check_connection_auth(runner) -> list[CheckResult]: # ───────────────────────────────────────────────────────────────────── -# DV-CONN-001 — Dataverse connection reference binding (S5.4, PASS/FAIL). +# DV-CONN-001 — Workday SOAP connection reference binding (S5.4, PASS/FAIL). # ───────────────────────────────────────────────────────────────────── @@ -378,67 +375,44 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.SKIPPED.value, description=_DV_CONN_DESC, result=( - "Dataverse token not available — skipping the Dataverse " - "connection-reference check." + "AgentBuilder client or active-agent botId not available — " + "skipping the Workday connection-reference check." ), )] - dv_refs = [ - r - for r in refs - if str(r.get("connectorid") or "").lower().endswith( - _DATAVERSE_CONNECTOR_SUFFIX - ) - and _is_dataverse_runtime_ref( - r.get("connectionreferencelogicalname") - ) - ] - if len(dv_refs) > 1: - names = ", ".join( - sorted( - str(ref.get("connectionreferencelogicalname") or "(unnamed)") - for ref in dv_refs - ) - ) - return [CheckResult(roles=_MAKER_ROLES, - checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.WARNING.value, - description=_DV_CONN_DESC, - result=( - "Multiple ESS Dataverse connection references match the " - f"runtime and legacy package fingerprints: {names}. " - "FlightCheck cannot determine which reference is active." - ), - remediation=( - "Remove obsolete Workday package references, then rerun " - "FlightCheck against the remaining Dataverse binding." - ), - doc_link=_DOC_SIMPLIFIED, - )] - dv_ref = dv_refs[0] if dv_refs else None + wd_ref = next( + ( + r + for r in refs + if str(r.get("connectorid") or "") + .lower() + .endswith(_WORKDAY_CONNECTOR_SUFFIX) + ), + None, + ) - if dv_ref is None: + if wd_ref is None: return [CheckResult(roles=_MAKER_ROLES, checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.NOT_CONFIGURED.value, + priority=Priority.HIGH.value, status=Status.FAILED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"(\u2026_{_DATAVERSE_REF_SUFFIX}, connector " - "shared_commondataserviceforapps) was not found in this " - "environment." + "The ESS Workday SOAP connection reference (connector " + "shared_workdaysoap) was not found in the Declarative Agent " + "components payload." ), remediation=( - "Install/repair the Workday extension pack so its Dataverse " - "connection reference is created, then bind it to a Dataverse " - "connection you own." + "Install or repair the Workday extension pack so its Workday " + "SOAP connection reference is created, then bind it to a " + "Workday connection you own." ), doc_link=_DOC_SIMPLIFIED, )] - dv_ref_name = str(dv_ref.get("connectionreferencelogicalname")) - connection_id = dv_ref.get("connectionid") - statuscode = dv_ref.get("statuscode") + wd_ref_name = str( + wd_ref.get("connectionreferencelogicalname") or "(unnamed)" + ) + connection_id = wd_ref.get("connectionid") if not connection_id: return [CheckResult(roles=_MAKER_ROLES, @@ -446,31 +420,13 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.FAILED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is unbound " - "(connectionid=null)." - ), - remediation=( - "In Power Platform / Copilot Studio, bind the Dataverse " - "connection reference to an active Dataverse connection owned " - "by your own account." - ), - doc_link=_DOC_SIMPLIFIED, - )] - - if statuscode != 1: - return [CheckResult(roles=_MAKER_ROLES, - checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.FAILED.value, - description=_DV_CONN_DESC, - result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is bound but inactive " - f"(statuscode={statuscode})." + "The ESS Workday SOAP connection reference " + f"({wd_ref_name}) is unbound (connectionId=null)." ), remediation=( - "Re-authenticate or re-bind the Dataverse connection so its " - "status is active, using an account you own." + "In Power Platform / Copilot Studio, bind the Workday SOAP " + "connection reference to an active Workday connection owned by " + "your own account." ), doc_link=_DOC_SIMPLIFIED, )] @@ -490,9 +446,8 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.PASSED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is bound to an active " - "connection." + owner_note + "The ESS Workday SOAP connection reference " + f"({wd_ref_name}) is bound to a connection." + owner_note ), doc_link=_DOC_SIMPLIFIED, )] diff --git a/solutions/ess-maker-skills/scripts/flightcheck/registry.py b/solutions/ess-maker-skills/scripts/flightcheck/registry.py index 6040d49ff..b5badccca 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/registry.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/registry.py @@ -521,15 +521,16 @@ class ResolvedPlan: priority=Priority.HIGH.value, roles=(Role.ESS_MAKER.value,), ), - # DV-CONN-001 — self-contained Dataverse read (its own connectionreferences - # query) plus a best-effort BAP owner echo. + # DV-CONN-001 — reads the Workday SOAP connection reference from the + # Declarative Agent minimalBots components API (AGENTBUILDER), plus a + # best-effort BAP owner echo (PP_ADMIN). CheckpointSpec( key="DV-CONN-001", category_fn=run_workday_extension_checks, category_label="Workday Extension", - clients=frozenset({DATAVERSE, PP_ADMIN}), + clients=frozenset({AGENTBUILDER, PP_ADMIN}), requires_config=True, - requires_dataverse_endpoint=True, + requires_dataverse_endpoint=False, priority=Priority.HIGH.value, roles=(Role.ESS_MAKER.value,), ), diff --git a/tests/flightcheck/checks/test_workday_extension.py b/tests/flightcheck/checks/test_workday_extension.py index 0e641716b..2910bb762 100644 --- a/tests/flightcheck/checks/test_workday_extension.py +++ b/tests/flightcheck/checks/test_workday_extension.py @@ -10,9 +10,9 @@ connection, degrades gracefully when it does not. Cached-ref read + a best-effort Power Platform admin owner echo — no cassette required (the admin connections listing is the ``validated`` pp_admin mock). - * DV-CONN-001 — PASS/FAIL/NOT_CONFIGURED/SKIPPED over a documented-tier - Dataverse ``connectionreferences`` read (stubbed with ``responses``); owner - echo via the ``validated`` pp_admin mock. + * DV-CONN-001 — PASS/FAIL/SKIPPED over the validated minimalBots components + read (Workday SOAP connection reference; faked ``runner.agentbuilder``); + owner echo via the ``validated`` pp_admin mock. * WD-REST-001 — pure-config check (restBaseUrl trimmed to '/api'). * WD-REST-002 — pure local-file check (user-context redirect topic); SKIPPED on the legacy install path. @@ -28,22 +28,18 @@ from dataclasses import dataclass, field from typing import Any -import responses - from tests.conftest import require_validated_mock +from tests.mocks import agentbuilder_connectivity as ab from tests.mocks import dataverse as dv from tests.mocks import pp_admin as pp +require_validated_mock(ab) require_validated_mock(dv) require_validated_mock(pp) from flightcheck.checks import workday_extension as wx # noqa: E402 from flightcheck.runner import Priority, Role, Status # noqa: E402 -_DV_CONNECTOR_ID = ( - "/providers/Microsoft.PowerApps/apis/shared_commondataserviceforapps" -) - # ───────────────────────────────────────────────────────────────────── # Minimal runner. The emitters read only these attributes; anything the @@ -62,12 +58,24 @@ def get_connections(self, _env_id: str): return self._connections +class _FakeAgentBuilder: + """Stand-in for FlightCheckRunner.agentbuilder. Only ``fetch_components`` + is consumed (DV-CONN-001's connection-reference read).""" + + def __init__(self, components: dict[str, Any]): + self._components = components + + def fetch_components(self, _agent_id: str): + return self._components + + @dataclass class _Runner: config: Any = field(default_factory=dict) env_url: str | None = None dv_token: str | None = None pp_admin: Any = None + agentbuilder: Any = None env_id: str | None = None _workday_connection_refs: list[dict[str, Any]] = field(default_factory=list) @@ -88,28 +96,6 @@ def _by_id(results): return {r.checkpoint_id: r for r in results} -def _dv_ref(*, connection_id, statuscode=1): - """A Dataverse connection reference matching the extension pack's shipped - ref (connector shared_commondataserviceforapps, logical-name suffix - 92b66).""" - return dv.connection_ref( - logical_name="msdyn_sharedcommondataserviceforapps_92b66", - display_name="Microsoft Dataverse", - connector_id=_DV_CONNECTOR_ID, - connection_id=connection_id, - statuscode=statuscode, - ) - - -def _register_refs(base_url: str, refs: list[dict[str, Any]]) -> None: - responses.add( - method="GET", - url=f"{base_url}/api/data/v9.2/connectionreferences", - json=dv.collection(refs), - status=200, - ) - - # ───────────────────────────────────────────────────────────────────── # WD-CONN-AUTH-001 — always MANUAL echo (S5.3). # ───────────────────────────────────────────────────────────────────── @@ -253,147 +239,88 @@ def test_never_passes_regardless_of_state(self): # ───────────────────────────────────────────────────────────────────── -# DV-CONN-001 — Dataverse connection binding (S5.4, PASS/FAIL). +# DV-CONN-001 — Workday SOAP connection binding (S5.4, PASS/FAIL). # ───────────────────────────────────────────────────────────────────── +def _runner_with_refs(references, *, pp_admin=None, env_id=None): + """A runner whose faked ``agentbuilder.fetch_components`` returns the given + connection references and whose config names an active agent (botId).""" + components = ab.components_with_references(references=references) + return _Runner( + config={"agent": {"botId": ab.MOCK_AGENT_ID}}, + agentbuilder=_FakeAgentBuilder(components), + pp_admin=pp_admin, + env_id=env_id, + ) + + class TestDataverseConnection: - @responses.activate - def test_bound_active_with_owner_echo_passes( - self, fake_dataverse_url, fake_token - ): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-active", statuscode=1)], - ) + def test_bound_with_owner_echo_passes(self): owner_conn = pp.connection( - name="dv-conn-active", - api_name="shared_commondataserviceforapps", + name="wd-conn-active", + api_name="shared_workdaysoap", extra_properties={"accountName": "maker@contoso.com"}, ) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id="wd-conn-active")], pp_admin=_FakePPAdmin([owner_conn]), env_id="env-1", ) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.PASSED.value - assert "bound to an active" in r.result + assert "bound to a connection" in r.result assert "maker@contoso.com" in r.result assert "your own account" in r.result - @responses.activate - def test_passes_without_pp_admin_notes_owner_unreadable( - self, fake_dataverse_url, fake_token - ): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-active", statuscode=1)], + def test_passes_without_pp_admin_notes_owner_unreadable(self): + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id="wd-conn-active")], ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.PASSED.value assert "owner could not be read" in r.result assert "your own account" in r.result - @responses.activate - def test_runtime_dataverse_reference_passes( - self, fake_dataverse_url, fake_token - ): - runtime_ref = dv.workday_connection_refs_runtime()[1] - _register_refs(fake_dataverse_url, [runtime_ref]) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, - ) - - r = _by_id( - wx.run_workday_extension_checks(runner) - )["DV-CONN-001"] - - assert r.status == Status.PASSED.value - assert ( - "msdyn_sharedcommondataserviceforapps_workdayruntime" - in r.result + def test_unbound_fails(self): + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id=None)], ) - - @responses.activate - def test_mixed_runtime_and_legacy_dataverse_refs_warn( - self, fake_dataverse_url, fake_token - ): - runtime_ref = dv.workday_connection_refs_runtime()[1] - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="legacy-dv"), runtime_ref], - ) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, - ) - - r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - - assert r.status == Status.WARNING.value - assert "Multiple ESS Dataverse connection references" in r.result - assert "Remove obsolete Workday package references" in r.remediation - - @responses.activate - def test_unbound_fails(self, fake_dataverse_url, fake_token): - _register_refs( - fake_dataverse_url, [_dv_ref(connection_id=None, statuscode=1)] - ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.FAILED.value assert "unbound" in r.result - assert "connectionid=null" in r.result - assert "bind the Dataverse connection reference" in r.remediation - - @responses.activate - def test_inactive_statuscode_fails(self, fake_dataverse_url, fake_token): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-inactive", statuscode=2)], - ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) + assert "connectionId=null" in r.result + assert "bind the Workday SOAP connection reference" in r.remediation + + def test_workday_ref_absent_fails(self): + # Only the default ServiceNow ref present — no Workday SOAP ref. + runner = _runner_with_refs(None) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.FAILED.value - assert "inactive" in r.result - assert "statuscode=2" in r.result - assert "Re-authenticate or re-bind" in r.remediation - - @responses.activate - def test_missing_ref_not_configured(self, fake_dataverse_url, fake_token): - # Only a Workday ref present — no Dataverse (92b66) ref. - _register_refs( - fake_dataverse_url, - [ - dv.connection_ref( - logical_name="new_sharedworkdaysoap_ff0df", - display_name="OAuthUser", - connector_id=dv.WORKDAY_SOAP_CONNECTOR_ID, - connection_id="wd-conn-1", - ) - ], - ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) + assert "was not found" in r.result + assert "shared_workdaysoap" in r.result + assert "Install or repair the Workday extension pack" in r.remediation + + def test_no_agentbuilder_client_skips(self): + runner = _Runner(config={"agent": {"botId": ab.MOCK_AGENT_ID}}) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.NOT_CONFIGURED.value - assert "was not found in this environment" in r.result - assert "Install/repair the Workday extension pack" in r.remediation + assert r.status == Status.SKIPPED.value + assert "not available" in r.result - def test_no_dv_token_skips(self): - runner = _Runner(env_url="https://x.crm.dynamics.com", dv_token="") + def test_no_active_agent_botid_skips(self): + runner = _Runner( + config={}, + agentbuilder=_FakeAgentBuilder(ab.components_with_references()), + ) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.SKIPPED.value - assert "Dataverse token not available" in r.result + assert "not available" in r.result # ───────────────────────────────────────────────────────────────────── diff --git a/tests/flightcheck/test_registry.py b/tests/flightcheck/test_registry.py index e8882af57..ca696abe2 100644 --- a/tests/flightcheck/test_registry.py +++ b/tests/flightcheck/test_registry.py @@ -294,7 +294,7 @@ class TestWorkdayExtensionCheckpoints: """skill-5 mints five checkpoints, all sharing checks/workday_extension.run_workday_extension_checks, category "Workday Extension". Two are always-MANUAL echoes/attestations, three are - programmatic (one Dataverse read + two pure-local).""" + programmatic (one minimalBots components read + two pure-local).""" _ALL = ( "WD-CONN-AUTH-001", @@ -329,10 +329,12 @@ def test_conn_auth_exact_beats_wd_conn_family(self): assert registry.resolve("WD-CONN-AUTH-001").key == "WD-CONN-AUTH-001" assert registry.resolve("WD-CONN-AUTH-001").is_family is False - def test_dv_conn_spec_declares_dataverse_and_pp_admin(self): + def test_dv_conn_spec_declares_agentbuilder_and_pp_admin(self): spec = registry.resolve("DV-CONN-001") - assert spec.clients == frozenset({registry.DATAVERSE, registry.PP_ADMIN}) - assert spec.requires_dataverse_endpoint is True + assert spec.clients == frozenset( + {registry.AGENTBUILDER, registry.PP_ADMIN} + ) + assert spec.requires_dataverse_endpoint is False assert spec.prereqs == () assert Role.ESS_MAKER.value in spec.roles @@ -354,7 +356,7 @@ def test_net_check_is_clientless_and_ppadmin_gated(self): def test_dv_conn_plan_unions_clients(self): plan = registry.transitive_requirements("DV-CONN-001") - assert registry.DATAVERSE in plan.clients + assert registry.AGENTBUILDER in plan.clients assert registry.PP_ADMIN in plan.clients def test_all_five_are_listable(self): From 671de3677b9f684b4d36259e47d9af34aaa43226 Mon Sep 17 00:00:00 2001 From: Dawn Jeong Date: Tue, 22 Sep 2026 15:07:25 -0700 Subject: [PATCH 3/6] flightcheck: fail loudly on malformed DV-CONN-001 changeset (review fix) Distinguish a genuinely-absent connectionReferenceChanges (missing key -> no references -> FAILED not-found) from a present-but-non-list payload (a shape we do not understand -> raise ValueError so the dispatcher degrades DV-CONN-001 to a WARNING). Mirrors native_agent._connection_references, the shipped precedent this check re-uses; the prior code collapsed both to [] and could report a confident "reference not found" on an unparseable 200 response. Adds a malformed-changeset test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../flightcheck/checks/workday_extension.py | 14 +++++++++++--- .../flightcheck/checks/test_workday_extension.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py index 0dc741a17..8ac4a47b1 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py @@ -184,8 +184,12 @@ def _query_connection_references(runner): and ``connectionReferenceChanges`` shape already back the shipped native ``DA-CONN-001`` check (``checks/native_agent.py``); see ``tests/fixtures/cassettes/INDEX.md`` and ``tests/mocks/ - agentbuilder_connectivity.py``. Any ``AgentBuilderHTTPError`` propagates so - the dispatcher degrades this checkpoint to a WARNING (fail loudly). + agentbuilder_connectivity.py``. Fails loudly (lets the dispatcher degrade + this checkpoint to a WARNING) rather than overclaiming: an + ``AgentBuilderHTTPError`` propagates, and a 200 payload whose + ``connectionReferenceChanges`` is present but not a list raises + ``ValueError`` (mirrors ``native_agent._connection_references``). A missing + changeset is treated as "no references" (genuine absence), not an error. """ client = getattr(runner, "agentbuilder", None) config = getattr(runner, "config", None) or {} @@ -194,8 +198,12 @@ def _query_connection_references(runner): return None changeset = client.fetch_components(agent_id) or {} changes = changeset.get("connectionReferenceChanges") - if not isinstance(changes, list): + if changes is None: return [] + if not isinstance(changes, list): + raise ValueError( + "Component fetch returned invalid connectionReferenceChanges." + ) refs = [] for change in changes: ref = (change or {}).get("connectionReference") or {} diff --git a/tests/flightcheck/checks/test_workday_extension.py b/tests/flightcheck/checks/test_workday_extension.py index 2910bb762..202372eb5 100644 --- a/tests/flightcheck/checks/test_workday_extension.py +++ b/tests/flightcheck/checks/test_workday_extension.py @@ -322,6 +322,22 @@ def test_no_active_agent_botid_skips(self): assert r.status == Status.SKIPPED.value assert "not available" in r.result + def test_malformed_changeset_degrades_to_warning(self): + # A 200 payload whose connectionReferenceChanges is present but not a + # list is a shape we do not understand: fail loudly (dispatcher WARNING) + # rather than reporting a confident "reference not found" FAILED. + runner = _Runner( + config={"agent": {"botId": ab.MOCK_AGENT_ID}}, + agentbuilder=_FakeAgentBuilder( + {"connectionReferenceChanges": {"unexpected": "dict"}} + ), + ) + r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] + + assert r.status == Status.WARNING.value + assert "Unable to run DV-CONN-001" in r.result + assert "DV-CONN-001" in r.remediation + # ───────────────────────────────────────────────────────────────────── # WD-REST-001 — REST base URL trimmed to /api (S5.5). From ab398f13739619cce437148ffa78ac1eab056061 Mon Sep 17 00:00:00 2001 From: Dawn Jeong Date: Tue, 22 Sep 2026 23:20:14 -0700 Subject: [PATCH 4/6] tests: cover DV-CONN-001 ServiceNow-present true-negative (was empty-list) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8246cd2c-37d0-4000-8fe3-fa9b082669e0 --- tests/flightcheck/checks/test_workday_extension.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/flightcheck/checks/test_workday_extension.py b/tests/flightcheck/checks/test_workday_extension.py index 202372eb5..3a4b2dfa1 100644 --- a/tests/flightcheck/checks/test_workday_extension.py +++ b/tests/flightcheck/checks/test_workday_extension.py @@ -296,8 +296,15 @@ def test_unbound_fails(self): assert "bind the Workday SOAP connection reference" in r.remediation def test_workday_ref_absent_fails(self): - # Only the default ServiceNow ref present — no Workday SOAP ref. - runner = _runner_with_refs(None) + # Only a ServiceNow ref is present - no Workday SOAP ref. + runner = _runner_with_refs( + [ + ab.connection_reference_change( + connector="shared_service-now", + connection_id="sn-1", + ) + ] + ) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.FAILED.value From ebcc343ff7c222514b08fce64490af3271bfc69c Mon Sep 17 00:00:00 2001 From: Dawn Jeong Date: Tue, 22 Sep 2026 20:58:49 -0700 Subject: [PATCH 5/6] flightcheck: re-point ESS-SOLN-001 to DA ALM configure (AB#7852510) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8246cd2c-37d0-4000-8fe3-fa9b082669e0 --- .../scripts/flightcheck/checks/solution.py | 335 ++++++++++++------ .../scripts/flightcheck/registry.py | 16 +- tests/flightcheck/checks/test_solution.py | 324 +++++++---------- .../flightcheck/test_cli_single_checkpoint.py | 18 - tests/flightcheck/test_registry.py | 28 +- 5 files changed, 388 insertions(+), 333 deletions(-) diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py index 0b721ea9c..2c754b91a 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py @@ -4,34 +4,40 @@ """ ESS FlightCheck — ESS Solution Installation Validation (ESS-SOLN-xxx) -Verifies that the base ESS agent solution has been installed into the target -Power Platform environment (skill-2 ``install-ess``). The install itself is a -manual AppSource / admin-center action; this module supplies the *programmatic -verification* that the solution landed, runnable in isolation via -``--checkpoint ESS-SOLN-001``. +Verifies that the base ESS declarative agent package is present in the target +Power Platform environment (skill-2 ``install-ess``). The check reads the +Copilot Studio minimalBots ALM configure surface so it works after the +Declarative Agent re-point away from Dataverse solution-table state. """ -from ..runner import CheckResult, Priority, Role, Status -from auth import query_all, AuthExpiredError # scripts/auth.py, on path via cli.py +from __future__ import annotations + +from typing import Any +from agentbuilder import ( + DEV_REALM, + PROD_REALM, + REALM_NAMES, + TEST_REALM, + AgentBuilderHTTPError, +) -# The AppSource "Employee Self Service" offer installs a managed solution whose -# unique name starts with this prefix. Three variants ship today — the base -# agent plus IT and HR editions — and skill-6 references the same namespace: -# msdyn_copilotforemployeeselfservice (base) -# msdyn_copilotforemployeeselfserviceit (IT) -# msdyn_copilotforemployeeselfservicehr (HR) -# A ``startswith`` match accepts whichever edition the tenant deployed (and any -# future variant) in a single round-trip. -_ESS_SOLUTION_PREFIX = "msdyn_copilotforemployeeselfservice" -_ESS_SOLN_FILTER = f"startswith(uniquename,'{_ESS_SOLUTION_PREFIX}')" -_ESS_SOLN_SELECT = "solutionid,uniquename,friendlyname,ismanaged,version" +from ..runner import CheckResult, Priority, Role, Status _ESS_SOLN_DOC_LINK = ( "https://learn.microsoft.com/en-us/microsoft-365/copilot/" "employee-self-service/install" ) -_ESS_SOLN_DESCRIPTION = "ESS base agent solution installed in the environment" +_ESS_SOLN_DESCRIPTION = "ESS base agent package present in the environment" +_DEFAULT_ALM_REALM = DEV_REALM +_ALM_NOT_OPTED_IN_ERROR_CODE = "4003" +_ALM_REALMS = { + "dev": DEV_REALM, + "development": DEV_REALM, + "test": TEST_REALM, + "prod": PROD_REALM, + "production": PROD_REALM, +} def run_solution_checks(runner) -> list[CheckResult]: @@ -45,100 +51,217 @@ def run_solution_checks(runner) -> list[CheckResult]: def _check_ess_solution_installed(runner) -> list[CheckResult]: - """ESS-SOLN-001: the base ESS solution is installed in the target env. + """ESS-SOLN-001: the base ESS agent package is installed in the target env.""" + agentbuilder = getattr(runner, "agentbuilder", None) + bot_id = _active_agent_bot_id(runner) + realm = _alm_realm(runner) - Always emits exactly one CheckResult (principle 7 — bucket multi-resource - findings). Never raises — all errors are caught and turned into WARNING - results so a transient Dataverse failure does not abort the whole - flightcheck run. - """ - env_url = getattr(runner, "env_url", None) - token = getattr(runner, "dv_token", None) - - if not env_url or not token: - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id="ESS-SOLN-001", category="Solution", - priority=Priority.CRITICAL.value, status=Status.SKIPPED.value, - description=_ESS_SOLN_DESCRIPTION, - result="Dataverse URL or access token not available in this run.", - doc_link=_ESS_SOLN_DOC_LINK, - )] + if agentbuilder is None: + return [ + _result( + Status.SKIPPED.value, + "AgentBuilder client not available in this run.", + ( + "Run FlightCheck with native AgentBuilder authentication " + "configured, then retry this check." + ), + ) + ] + if not bot_id: + return [ + _result( + Status.SKIPPED.value, + "No agent botId is recorded in .local/config.json.", + "Run setup so the agent botId is recorded, then retry this check.", + ) + ] + if realm is None: + return [ + _result( + Status.FAILED.value, + "The configured ALM realm is not Dev, Test, or Prod.", + ( + "Set almRealm, grsRealm, or minimalBotsAlmRealm to Dev, " + "Test, or Prod, then retry this check." + ), + ) + ] try: - solutions = query_all( - env_url, token, - "solutions", - _ESS_SOLN_SELECT, - _ESS_SOLN_FILTER, - ) + config = agentbuilder.get_realm_configuration(bot_id, realm) + except AgentBuilderHTTPError as exc: + if _is_not_opted_into_alm_error(exc): + return [_not_opted_into_alm_result(bot_id)] + return [ + _result( + Status.WARNING.value, + ( + "Unable to verify ESS package state from AgentBuilder ALM " + f"configure: {exc}" + ), + ( + "Retry the read-only ALM configure check after verifying " + "Copilot Studio AgentBuilder access and service availability." + ), + ) + ] + except Exception as exc: + return [ + _result( + Status.WARNING.value, + ( + "Unable to verify ESS package state from AgentBuilder ALM " + f"configure: {type(exc).__name__}: {exc}" + ), + ( + "Retry the read-only ALM configure check after verifying " + "Copilot Studio AgentBuilder access and service availability." + ), + ) + ] - if not solutions: - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id="ESS-SOLN-001", category="Solution", - priority=Priority.CRITICAL.value, status=Status.FAILED.value, - description=_ESS_SOLN_DESCRIPTION, - result=( - "No solution whose unique name starts with " - f"'{_ESS_SOLUTION_PREFIX}' is installed in this " - "environment. The base ESS agent is not present." + grs_repository_id = str(config.get("grsRepositoryId") or "").strip() + commit_sha = str(config.get("commitSha") or "").strip() + if not grs_repository_id or not commit_sha: + return [ + _result( + Status.FAILED.value, + ( + "AgentBuilder ALM configure did not return both " + "grsRepositoryId and commitSha. The ESS base package is " + "not present in the agent's GRS package state." ), - remediation=( - "Install the Employee Self Service agent from AppSource " - "into this environment (Microsoft 365 admin center / " - "AppSource -> get 'Employee Self Service' -> deploy to the " - "target environment), wait for the solution import to " - "finish, then re-run this check." + ( + "Install or import the Employee Self Service base package " + "for this agent, confirm the ALM configure response has " + "a repository and commit, then retry this check." ), - doc_link=_ESS_SOLN_DOC_LINK, - )] + ) + ] - installed = ", ".join( - _describe_solution(s) - for s in sorted(solutions, key=lambda s: s.get("uniquename", "")) - ) - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id="ESS-SOLN-001", category="Solution", - priority=Priority.CRITICAL.value, status=Status.PASSED.value, - description=_ESS_SOLN_DESCRIPTION, - result=f"ESS base agent solution installed: {installed}.", - doc_link=_ESS_SOLN_DOC_LINK, - )] - - except AuthExpiredError as e: - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id="ESS-SOLN-001", category="Solution", - priority=Priority.CRITICAL.value, status=Status.WARNING.value, - description=_ESS_SOLN_DESCRIPTION, - result=str(e), - remediation="Re-run FlightCheck to refresh the access token.", - doc_link=_ESS_SOLN_DOC_LINK, - )] - except Exception as e: - # Per principle 3 (fail loudly): surface unexpected Dataverse failures - # as WARNING rather than silently passing. Surface the HTTP status - # code when available so a 403 (insufficient privileges) is - # distinguishable from a 5xx (transient) at a glance. - status_code = getattr(getattr(e, "response", None), "status_code", None) - status_hint = f" [HTTP {status_code}]" if status_code is not None else "" - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id="ESS-SOLN-001", category="Solution", - priority=Priority.CRITICAL.value, status=Status.WARNING.value, - description=_ESS_SOLN_DESCRIPTION, - result=( - f"Unable to verify the ESS solution: " - f"{type(e).__name__}{status_hint}: {e}" - ), - remediation=( - "Inspect the error above; common causes are insufficient " - "Dataverse privileges on the solutions table (typically " - "surfaces as HTTP 403) or a transient platform error (HTTP 5xx)." + return [ + _result( + Status.PASSED.value, + ( + "ESS base package is present in GRS: " + f"repository {grs_repository_id}, commit {commit_sha}." ), - doc_link=_ESS_SOLN_DOC_LINK, - )] + ) + ] + + +def _result( + status: str, + result: str, + remediation: str = "", +) -> CheckResult: + return CheckResult( + checkpoint_id="ESS-SOLN-001", + category="Solution", + priority=Priority.CRITICAL.value, + status=status, + description=_ESS_SOLN_DESCRIPTION, + result=result, + remediation=remediation, + doc_link=_ESS_SOLN_DOC_LINK, + roles=[Role.ESS_MAKER.value], + ) + + +def _active_agent_bot_id(runner) -> str | None: + config = getattr(runner, "config", None) or {} + agents = config.get("agents") or [] + active_slug = config.get("activeAgent") or (config.get("agent") or {}).get( + "slug" + ) + if active_slug: + for agent in agents: + if isinstance(agent, dict) and agent.get("slug") == active_slug: + bot_id = str(agent.get("botId") or "").strip() + if bot_id: + return bot_id + single = config.get("agent") or {} + bot_id = str(single.get("botId") or "").strip() + if bot_id: + return bot_id + for agent in agents: + if isinstance(agent, dict): + bot_id = str(agent.get("botId") or "").strip() + if bot_id: + return bot_id + return None + + +def _alm_realm(runner) -> int | None: + config = getattr(runner, "config", None) or {} + single = config.get("agent") or {} + raw = ( + config.get("minimalBotsAlmRealm") + or config.get("grsRealm") + or config.get("almRealm") + or single.get("minimalBotsAlmRealm") + or single.get("grsRealm") + or single.get("almRealm") + or single.get("realm") + or _DEFAULT_ALM_REALM + ) + if type(raw) is int and raw in REALM_NAMES: + return raw + if isinstance(raw, str): + text = raw.strip() + if text.isdigit(): + realm = int(text) + return realm if realm in REALM_NAMES else None + return _ALM_REALMS.get(text.casefold()) + return None + + +def _is_not_opted_into_alm_error(error: AgentBuilderHTTPError) -> bool: + if _error_code_matches(getattr(error, "error_code", None)): + return True + response = getattr(error, "response", None) + if response is None: + return False + try: + body = response.json() + except ValueError: + return False + return _payload_is_not_opted_into_alm(body) + + +def _payload_is_not_opted_into_alm(payload: Any) -> bool: + if not isinstance(payload, dict): + return False + candidates = [ + payload.get("ErrorCode"), + payload.get("errorCode"), + payload.get("code"), + ] + for key in ("error", "Error"): + nested = payload.get(key) + if isinstance(nested, dict): + candidates.extend( + [nested.get("code"), nested.get("Code"), nested.get("ErrorCode")] + ) + else: + candidates.append(nested) + return any(_error_code_matches(candidate) for candidate in candidates) + + +def _error_code_matches(value: Any) -> bool: + return str(value).strip() == _ALM_NOT_OPTED_IN_ERROR_CODE -def _describe_solution(sol: dict) -> str: - """Render one solution row as ``uniquename (vX.Y.Z)`` for the result text.""" - name = sol.get("uniquename", "") - version = sol.get("version") - return f"{name} (v{version})" if version else name +def _not_opted_into_alm_result(bot_id: str) -> CheckResult: + return _result( + Status.NOT_CONFIGURED.value, + ( + f"Agent {bot_id} is not opted into ALM. AgentBuilder ALM configure " + "returned error code 4003, so FlightCheck cannot read its GRS " + "package state." + ), + ( + "Opt the agent into Application Lifecycle Management (ALM) in " + "Copilot Studio, then retry this check." + ), + ) diff --git a/solutions/ess-maker-skills/scripts/flightcheck/registry.py b/solutions/ess-maker-skills/scripts/flightcheck/registry.py index b5badccca..8771babfa 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/registry.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/registry.py @@ -247,22 +247,16 @@ class ResolvedPlan: roles=(Role.POWER_PLATFORM_ADMIN.value,), ), # ---- Solution: ESS-SOLN-001 (skill-2 install-ess) ---- - # ESS-SOLN-001: the base ESS agent solution (msdyn_copilotforemployeeselfservice*) - # is installed in the target env. Queries the Dataverse `solutions` table - # (DATAVERSE client, already wired in cli.py's single-checkpoint path — no - # new client init). Prereq ENV-002 (Dataverse provisioned) transitively - # pulls ENV-001 (environment exists). Environment Maker owns the fix; the - # AppSource install itself is a manual portal action, but this check - # definitively verifies the outcome, so the S2.1 checklist row auto-completes - # (`prog` gate) on a PASSED result. + # ESS-SOLN-001: the base ESS DA package is present in the agent's GRS/ALM + # state. Reads the AgentBuilder minimalBots ALM configure API instead of + # Dataverse solution-table state. CheckpointSpec( key="ESS-SOLN-001", category_fn=run_solution_checks, category_label="Solution", - clients=frozenset({DATAVERSE}), + clients=frozenset({AGENTBUILDER}), requires_config=True, - requires_dataverse_endpoint=True, - prereqs=("ENV-002",), + requires_dataverse_endpoint=False, priority=Priority.CRITICAL.value, roles=(Role.ESS_MAKER.value,), ), diff --git a/tests/flightcheck/checks/test_solution.py b/tests/flightcheck/checks/test_solution.py index 4e28de6b3..094a91a64 100644 --- a/tests/flightcheck/checks/test_solution.py +++ b/tests/flightcheck/checks/test_solution.py @@ -1,229 +1,179 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""End-to-end tests for ESS-SOLN-001 (ESS base agent solution installed) in -``solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py``. - -Mocks the single Dataverse Web API endpoint the check calls (the ``solutions`` -table query) with the ``responses`` library, then invokes the real production -helper ``_check_ess_solution_installed`` and asserts on the resulting -``CheckResult``. - -Mock backing: Dataverse Web API v9.2 is the ``documented`` tier per -``tests/fixtures/cassettes/INDEX.md`` — no cassette required. The ``solutions`` -response shape comes from the MS Learn entity reference: -https://learn.microsoft.com/power-apps/developer/data-platform/reference/entities/solution +"""Tests for ESS-SOLN-001's Declarative Agent ALM configure read. + +The check consumes the validated AgentBuilder Minimal Bot API +``GET /copilotstudio/minimalBots/alm/{agent_id}/configure`` shape captured in +``tests/fixtures/cassettes/agentbuilder_readiness.yaml``. """ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any import pytest -import responses -from tests.conftest import FAKE_DATAVERSE_URL, require_validated_mock -from tests.mocks import dataverse as dv +from agentbuilder import AgentBuilderHTTPError, DEV_REALM +from flightcheck.checks.solution import _check_ess_solution_installed +from flightcheck.runner import Status +from tests.conftest import require_validated_mock +from tests.mocks import agentbuilder_connectivity as ab -require_validated_mock(dv) +require_validated_mock(ab) -# Production module — flightcheck is importable because pyproject.toml puts -# solutions/ess-maker-skills/scripts on pythonpath. -from flightcheck.checks.solution import _check_ess_solution_installed # noqa: E402 +VALIDATED_COMMIT_SHA = "4bc80d2768da5de930fd56a1f5ee815b8f9d1d3b" -BASE_URL = FAKE_DATAVERSE_URL +class _FakeResponse: + def __init__(self, payload: dict[str, Any]): + self.status_code = 400 + self._payload = payload -# Verbatim from the production check; if these drift, mock-builder URLs will -# stop matching and tests fail loudly with an unregistered-URL error. -ESS_SOLN_SELECT = "solutionid,uniquename,friendlyname,ismanaged,version" -ESS_SOLN_FILTER = "startswith(uniquename,'msdyn_copilotforemployeeselfservice')" + def json(self) -> dict[str, Any]: + return self._payload -SOLUTION_ID = "11111111-1111-1111-1111-111111111111" +class _FakeAgentBuilder: + def __init__( + self, + payload: dict[str, Any] | None = None, + *, + error: Exception | None = None, + ) -> None: + self._payload = payload or _configuration_with_commit() + self._error = error + self.calls: list[tuple[str, int]] = [] -# ─────────────────────────────────────────────────────────────────────── -# Minimal runner for solution-check tests. -# ─────────────────────────────────────────────────────────────────────── + def get_realm_configuration(self, agent_id: str, realm: int) -> dict[str, Any]: + self.calls.append((agent_id, realm)) + if self._error is not None: + raise self._error + return self._payload @dataclass -class _MinimalRunner: - env_url: str | None - dv_token: str | None - - -@pytest.fixture -def runner(fake_dataverse_url: str, fake_token: str) -> _MinimalRunner: - return _MinimalRunner(env_url=fake_dataverse_url, dv_token=fake_token) - +class _Runner: + agentbuilder: Any = None + config: dict[str, Any] = field(default_factory=dict) -# ─────────────────────────────────────────────────────────────────────── -# Mock payload builders + registration helpers -# ─────────────────────────────────────────────────────────────────────── +def _configuration_with_commit(**overrides: Any) -> dict[str, Any]: + payload = ab.configuration() + payload["commitSha"] = VALIDATED_COMMIT_SHA + payload.update(overrides) + return payload -def _solution_record( - uniquename: str, - *, - version: str = "1.0.0.0", - ismanaged: bool = True, -) -> dict[str, Any]: - """One ``solutions`` row matching the production $select. - Field naming per - https://learn.microsoft.com/power-apps/developer/data-platform/reference/entities/solution - """ +def _config(*, bot_id: str | None = ab.MOCK_AGENT_ID) -> dict[str, Any]: + agent: dict[str, Any] = {"slug": "ess-dev"} + if bot_id is not None: + agent["botId"] = bot_id return { - "@odata.etag": 'W/"1"', - "solutionid": SOLUTION_ID, - "uniquename": uniquename, - "friendlyname": uniquename, - "ismanaged": ismanaged, - "version": version, + "activeAgent": "ess-dev", + "agent": agent.copy(), + "agents": [agent], } -def _register_solutions(solutions: list[dict[str, Any]]) -> None: - responses.add(**dv.query( - base_url=BASE_URL, - entity_set="solutions", - records=solutions, - select=ESS_SOLN_SELECT, - filter_expr=ESS_SOLN_FILTER, - )) +def test_passes_when_configure_returns_grs_repository_and_commit() -> None: + agentbuilder = _FakeAgentBuilder() + runner = _Runner(agentbuilder=agentbuilder, config=_config()) + + result = _check_ess_solution_installed(runner)[0] + + assert result.status == Status.PASSED.value + assert "ESS base package is present in GRS" in result.result + assert ab.MOCK_FAMILY_ID in result.result + assert VALIDATED_COMMIT_SHA in result.result + assert result.remediation == "" + assert agentbuilder.calls == [(ab.MOCK_AGENT_ID, DEV_REALM)] + + +@pytest.mark.parametrize( + ("field", "result_phrase"), + [ + ("grsRepositoryId", "grsRepositoryId"), + ("commitSha", "commitSha"), + ], +) +def test_fails_when_configure_omits_required_package_state( + field: str, + result_phrase: str, +) -> None: + payload = _configuration_with_commit() + payload[field] = "" + runner = _Runner(agentbuilder=_FakeAgentBuilder(payload), config=_config()) + + result = _check_ess_solution_installed(runner)[0] + + assert result.status == Status.FAILED.value + assert result_phrase in result.result + assert "not present in the agent's GRS package state" in result.result + assert "Install or import the Employee Self Service base package" in ( + result.remediation + ) + assert "confirm the ALM configure response has a repository and commit" in ( + result.remediation + ) -# ─────────────────────────────────────────────────────────────────────── -# Tests — one per verdict path. -# ─────────────────────────────────────────────────────────────────────── +def test_not_configured_when_agent_is_not_opted_into_alm() -> None: + error = AgentBuilderHTTPError( + "Dev realm configuration", + 400, + response=_FakeResponse({"ErrorCode": 4003}), + ) + runner = _Runner( + agentbuilder=_FakeAgentBuilder(error=error), + config=_config(), + ) + result = _check_ess_solution_installed(runner)[0] -def test_skipped_when_env_url_missing() -> None: - results = _check_ess_solution_installed( - _MinimalRunner(env_url=None, dv_token="tok") + assert result.status == Status.NOT_CONFIGURED.value + assert "not opted into ALM" in result.result + assert "error code 4003" in result.result + assert "Opt the agent into Application Lifecycle Management" in ( + result.remediation ) - assert len(results) == 1 - r = results[0] - assert r.checkpoint_id == "ESS-SOLN-001" - assert r.category == "Solution" - assert r.status == "Skipped" - assert "Dataverse URL or access token not available" in r.result + assert "Copilot Studio" in result.remediation -def test_skipped_when_token_missing() -> None: - results = _check_ess_solution_installed( - _MinimalRunner(env_url=BASE_URL, dv_token=None) - ) - assert results[0].status == "Skipped" - - -@responses.activate -def test_failed_when_no_ess_solution(runner: _MinimalRunner) -> None: - _register_solutions(solutions=[]) - - results = _check_ess_solution_installed(runner) - assert len(results) == 1 - r = results[0] - assert r.checkpoint_id == "ESS-SOLN-001" - assert r.status == "Failed" - assert "not present" in r.result - assert "AppSource" in r.remediation - - -@responses.activate -def test_passed_when_base_solution_present(runner: _MinimalRunner) -> None: - _register_solutions(solutions=[ - _solution_record("msdyn_copilotforemployeeselfservice", version="1.2.3.4"), - ]) - - results = _check_ess_solution_installed(runner) - assert len(results) == 1 - r = results[0] - assert r.checkpoint_id == "ESS-SOLN-001" - assert r.status == "Passed" - assert "msdyn_copilotforemployeeselfservice" in r.result - assert "1.2.3.4" in r.result - # Principle 8: PASSED carries no remediation. - assert r.remediation == "" - - -@responses.activate -def test_passed_when_it_variant_present(runner: _MinimalRunner) -> None: - _register_solutions(solutions=[ - _solution_record("msdyn_copilotforemployeeselfserviceit"), - ]) - - r = _check_ess_solution_installed(runner)[0] - assert r.status == "Passed" - assert "msdyn_copilotforemployeeselfserviceit" in r.result - - -@responses.activate -def test_passed_when_hr_variant_present(runner: _MinimalRunner) -> None: - _register_solutions(solutions=[ - _solution_record("msdyn_copilotforemployeeselfservicehr"), - ]) - - r = _check_ess_solution_installed(runner)[0] - assert r.status == "Passed" - assert "msdyn_copilotforemployeeselfservicehr" in r.result - - -@responses.activate -def test_passed_lists_multiple_editions(runner: _MinimalRunner) -> None: - _register_solutions(solutions=[ - _solution_record("msdyn_copilotforemployeeselfservice"), - _solution_record("msdyn_copilotforemployeeselfserviceit"), - ]) - - r = _check_ess_solution_installed(runner)[0] - assert r.status == "Passed" - assert "msdyn_copilotforemployeeselfservice " in r.result - assert "msdyn_copilotforemployeeselfserviceit" in r.result - - -@responses.activate -def test_warning_when_dataverse_returns_500(runner: _MinimalRunner) -> None: - """A transient platform error must surface as WARNING, not silently PASS.""" - responses.add( - "GET", - dv.build_query_url( - BASE_URL, - "solutions", - select=ESS_SOLN_SELECT, - filter_expr=ESS_SOLN_FILTER, - ), - json={"error": {"code": "0x80040220", "message": "boom"}}, - status=500, +def test_skips_when_agentbuilder_client_is_missing() -> None: + result = _check_ess_solution_installed(_Runner(config=_config()))[0] + + assert result.status == Status.SKIPPED.value + assert "AgentBuilder client not available" in result.result + assert "native AgentBuilder authentication" in result.remediation + + +def test_skips_when_bot_id_is_missing() -> None: + runner = _Runner( + agentbuilder=_FakeAgentBuilder(), + config=_config(bot_id=None), ) - r = _check_ess_solution_installed(runner)[0] - assert r.status == "Warning" - assert "Unable to verify the ESS solution" in r.result - - -@responses.activate -def test_warning_when_dataverse_returns_401(runner: _MinimalRunner) -> None: - """A 401 must surface as WARNING with an auth-expired hint. - - Exercises the AuthExpiredError catch block in _check_ess_solution_installed. - """ - responses.add( - "GET", - dv.build_query_url( - BASE_URL, - "solutions", - select=ESS_SOLN_SELECT, - filter_expr=ESS_SOLN_FILTER, - ), - json={"error": {"code": "0x80048306", "message": "token expired"}}, - status=401, + result = _check_ess_solution_installed(runner)[0] + + assert result.status == Status.SKIPPED.value + assert "No agent botId" in result.result + assert "Run setup" in result.remediation + + +def test_warning_when_configure_read_errors() -> None: + error = AgentBuilderHTTPError("Dev realm configuration", 500) + runner = _Runner( + agentbuilder=_FakeAgentBuilder(error=error), + config=_config(), ) - r = _check_ess_solution_installed(runner)[0] - assert r.status == "Warning" - assert "401" in r.result - assert "Re-run FlightCheck" in r.remediation + result = _check_ess_solution_installed(runner)[0] + + assert result.status == Status.WARNING.value + assert "Unable to verify ESS package state" in result.result + assert "Dev realm configuration failed with HTTP 500" in result.result + assert "Retry the read-only ALM configure check" in result.remediation + assert "AgentBuilder access" in result.remediation diff --git a/tests/flightcheck/test_cli_single_checkpoint.py b/tests/flightcheck/test_cli_single_checkpoint.py index 95f032393..969c602ed 100644 --- a/tests/flightcheck/test_cli_single_checkpoint.py +++ b/tests/flightcheck/test_cli_single_checkpoint.py @@ -113,24 +113,6 @@ def test_missing_config_exits_1( cli._run_single_checkpoint(_args("ESS-SOLN-001", tmp_path)) assert exc.value.code == 1 - def test_missing_dataverse_endpoint_exits_1( - self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch - ) -> None: - # Config present (so the config gate passes) but no dataverseEndpoint, - # and ESS-SOLN-001 requires one -> the endpoint gate fires, still - # before any auth. - plan = registry.transitive_requirements("ESS-SOLN-001") - assert plan.requires_dataverse_endpoint, ( - "test assumes ESS-SOLN-001 requires a Dataverse endpoint" - ) - local = tmp_path / ".local" - local.mkdir() - (local / "config.json").write_text("{}", encoding="utf-8") - monkeypatch.chdir(tmp_path) - with pytest.raises(SystemExit) as exc: - cli._run_single_checkpoint(_args("ESS-SOLN-001", tmp_path)) - assert exc.value.code == 1 - def test_capacity_uses_explicit_environment_id_without_dataverse( self, tmp_path: Path, diff --git a/tests/flightcheck/test_registry.py b/tests/flightcheck/test_registry.py index ca696abe2..4a9a07460 100644 --- a/tests/flightcheck/test_registry.py +++ b/tests/flightcheck/test_registry.py @@ -207,6 +207,17 @@ def test_native_agent_checkpoints_use_only_native_read_clients(self): "Native Agent" ] + def test_ess_soln_uses_agentbuilder_without_dataverse(self): + spec = registry.resolve("ESS-SOLN-001") + assert spec is not None and spec.key == "ESS-SOLN-001" + assert spec.clients == frozenset({registry.AGENTBUILDER}) + assert spec.requires_dataverse_endpoint is False + + plan = registry.transitive_requirements("ESS-SOLN-001") + assert plan.clients == frozenset({registry.AGENTBUILDER}) + assert plan.requires_config is True + assert plan.requires_dataverse_endpoint is False + def test_env009_is_individually_targetable_with_dataverse_only(self): spec = registry.resolve("ENV-009") assert spec is not None and spec.key == "ENV-009" @@ -216,24 +227,19 @@ def test_env009_is_individually_targetable_with_dataverse_only(self): assert plan.requires_dataverse_endpoint is True assert len(plan.ordered_fns) == 1 - def test_ess_soln_001_resolves_and_pulls_env_prereqs(self): + def test_ess_soln_001_resolves_to_agentbuilder_configure_read(self): spec = registry.resolve("ESS-SOLN-001") assert spec is not None and spec.key == "ESS-SOLN-001" assert spec.category_label == "Solution" assert spec.category_fn is run_solution_checks - # Solution presence is a pure Dataverse read. - assert spec.clients == frozenset({registry.DATAVERSE}) - assert spec.prereqs == ("ENV-002",) + assert spec.clients == frozenset({registry.AGENTBUILDER}) + assert spec.prereqs == () plan = registry.transitive_requirements("ESS-SOLN-001") - assert registry.DATAVERSE in plan.clients + assert plan.clients == frozenset({registry.AGENTBUILDER}) assert plan.requires_config is True - assert plan.requires_dataverse_endpoint is True - # Own fn (run_solution_checks) plus the shared run_environment_checks - # that ENV-001+ENV-002 pull in -> exactly two, environment first. + assert plan.requires_dataverse_endpoint is False fns = [fn for _label, fn in plan.ordered_fns] - assert run_solution_checks in fns - assert len(fns) == 2 - assert fns.index(run_solution_checks) == len(fns) - 1 + assert fns == [run_solution_checks] class TestListCheckpoints: From 197464c5086c7e49399be6b1ec57af9c5519d8cf Mon Sep 17 00:00:00 2001 From: Dawn Jeong Date: Tue, 22 Sep 2026 23:20:32 -0700 Subject: [PATCH 6/6] flightcheck: scope ESS-SOLN-001 PASS text to observed GRS state and add realm-branch tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 8246cd2c-37d0-4000-8fe3-fa9b082669e0 --- .../scripts/flightcheck/checks/solution.py | 5 +- tests/flightcheck/checks/test_solution.py | 65 ++++++++++++++----- tests/mocks/agentbuilder_connectivity.py | 4 +- 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py index 2c754b91a..abdb13fb3 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/solution.py @@ -143,8 +143,9 @@ def _check_ess_solution_installed(runner) -> list[CheckResult]: _result( Status.PASSED.value, ( - "ESS base package is present in GRS: " - f"repository {grs_repository_id}, commit {commit_sha}." + "Agent has a committed GRS package " + f"(repository {grs_repository_id}, commit {commit_sha}); " + "ESS base-package identity match pending US 7792604." ), ) ] diff --git a/tests/flightcheck/checks/test_solution.py b/tests/flightcheck/checks/test_solution.py index 094a91a64..5547fc054 100644 --- a/tests/flightcheck/checks/test_solution.py +++ b/tests/flightcheck/checks/test_solution.py @@ -15,8 +15,8 @@ import pytest -from agentbuilder import AgentBuilderHTTPError, DEV_REALM -from flightcheck.checks.solution import _check_ess_solution_installed +from agentbuilder import AgentBuilderHTTPError, DEV_REALM, PROD_REALM, TEST_REALM +from flightcheck.checks.solution import _alm_realm, _check_ess_solution_installed from flightcheck.runner import Status from tests.conftest import require_validated_mock from tests.mocks import agentbuilder_connectivity as ab @@ -24,9 +24,6 @@ require_validated_mock(ab) -VALIDATED_COMMIT_SHA = "4bc80d2768da5de930fd56a1f5ee815b8f9d1d3b" - - class _FakeResponse: def __init__(self, payload: dict[str, Any]): self.status_code = 400 @@ -43,7 +40,7 @@ def __init__( *, error: Exception | None = None, ) -> None: - self._payload = payload or _configuration_with_commit() + self._payload = payload or ab.configuration() self._error = error self.calls: list[tuple[str, int]] = [] @@ -60,13 +57,6 @@ class _Runner: config: dict[str, Any] = field(default_factory=dict) -def _configuration_with_commit(**overrides: Any) -> dict[str, Any]: - payload = ab.configuration() - payload["commitSha"] = VALIDATED_COMMIT_SHA - payload.update(overrides) - return payload - - def _config(*, bot_id: str | None = ab.MOCK_AGENT_ID) -> dict[str, Any]: agent: dict[str, Any] = {"slug": "ess-dev"} if bot_id is not None: @@ -85,9 +75,10 @@ def test_passes_when_configure_returns_grs_repository_and_commit() -> None: result = _check_ess_solution_installed(runner)[0] assert result.status == Status.PASSED.value - assert "ESS base package is present in GRS" in result.result - assert ab.MOCK_FAMILY_ID in result.result - assert VALIDATED_COMMIT_SHA in result.result + assert "Agent has a committed GRS package" in result.result + assert "ESS base-package identity match pending US 7792604" in result.result + assert ab.MOCK_ENV_ID in result.result + assert ab.MOCK_COMMIT_SHA in result.result assert result.remediation == "" assert agentbuilder.calls == [(ab.MOCK_AGENT_ID, DEV_REALM)] @@ -103,7 +94,7 @@ def test_fails_when_configure_omits_required_package_state( field: str, result_phrase: str, ) -> None: - payload = _configuration_with_commit() + payload = ab.configuration() payload[field] = "" runner = _Runner(agentbuilder=_FakeAgentBuilder(payload), config=_config()) @@ -163,6 +154,46 @@ def test_skips_when_bot_id_is_missing() -> None: assert "Run setup" in result.remediation +def test_fails_when_configured_alm_realm_is_not_supported() -> None: + config = _config() + config["almRealm"] = "sandbox" + agentbuilder = _FakeAgentBuilder() + runner = _Runner(agentbuilder=agentbuilder, config=config) + + result = _check_ess_solution_installed(runner)[0] + + assert result.status == Status.FAILED.value + assert "configured ALM realm is not Dev, Test, or Prod" in result.result + assert "Set almRealm, grsRealm, or minimalBotsAlmRealm" in ( + result.remediation + ) + assert "Dev, Test, or Prod" in result.remediation + assert agentbuilder.calls == [] + + +@pytest.mark.parametrize( + ("raw_realm", "expected"), + [ + (DEV_REALM, DEV_REALM), + ("dev", DEV_REALM), + ("DEV", DEV_REALM), + ("test", TEST_REALM), + ("TeSt", TEST_REALM), + ("prod", PROD_REALM), + ("PROD", PROD_REALM), + (str(TEST_REALM), TEST_REALM), + ("sandbox", None), + ], +) +def test_alm_realm_maps_supported_config_values( + raw_realm: Any, + expected: int | None, +) -> None: + runner = _Runner(config={"almRealm": raw_realm}) + + assert _alm_realm(runner) == expected + + def test_warning_when_configure_read_errors() -> None: error = AgentBuilderHTTPError("Dev realm configuration", 500) runner = _Runner( diff --git a/tests/mocks/agentbuilder_connectivity.py b/tests/mocks/agentbuilder_connectivity.py index f666f928c..01cc429da 100644 --- a/tests/mocks/agentbuilder_connectivity.py +++ b/tests/mocks/agentbuilder_connectivity.py @@ -22,6 +22,7 @@ MOCK_FAMILY_ID = "00000000-0000-0000-0000-000000003333" MOCK_CONNECTION_ID = "mock-servicenow-connection" MOCK_WORKDAY_CONNECTION_ID = "mock-workday-connection" +MOCK_COMMIT_SHA = "4bc80d2768da5de930fd56a1f5ee815b8f9d1d3b" MOCK_AGENTBUILDER_BASE = ( "https://00000000000000000000000000000000." "0.environment.api.test.powerplatform.com" @@ -42,7 +43,8 @@ def configuration() -> dict[str, Any]: "realm": "Dev", "cdsBotId": MOCK_AGENT_ID, "schemaName": "gptagent_mockemployeeselfservice", - "grsRepositoryId": MOCK_FAMILY_ID, + "grsRepositoryId": MOCK_ENV_ID, + "commitSha": MOCK_COMMIT_SHA, }