diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/_agent_connection_refs.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/_agent_connection_refs.py deleted file mode 100644 index 5652645a1..000000000 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/_agent_connection_refs.py +++ /dev/null @@ -1,242 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -"""ENV-004 helper — resolve the connection references an ESS agent uses. - -ENV-004 (``environment.py:_check_connections_and_refs``) historically -judged **every** ``connectionreference`` row in the environment. That -produced false FAILs on refs an ESS agent never touches — most visibly -the ESS-shipped placeholder refs that ship *unbound by design* on a -Workday **simplified** install (``msdyn_Dataverse`` / -``msdyn_ContentConversion`` with ``connectionid = null``). A customer -running any other Power Platform app in the same environment would also -see that app's refs judged against ESS's expectations. - -This module builds the **allow-list of connection references the agent -actually uses** so ENV-004 can scope its verdict to them. The chain: - - config agent botId(s) - -> Dataverse ``botcomponents`` (enabled topics: componenttype 9, - statecode 0) ``data`` column (the topic YAML) - -> extract every ``flowId:`` referenced by an InvokeFlowAction - -> Power Platform Admin ``pp.get_flow(env_id, flow_id)`` (BAP - per-flow detail) for each discovered flowId - -> read ``properties.connectionReferences[*]`` from that detail for - its ``connectionReferenceLogicalName`` (the allow-list) and its - connector (for scoping the unbound-connection branch). - -Why this join is sound (the flowId↔flow identity linkage): - A Power Automate cloud flow's identity is a single GUID that is the - same across surfaces — the topic's ``InvokeFlowAction.flowId``, the - Dataverse ``workflow.workflowid``, and the BAP flow detail endpoint's - ``/flows/{id}`` key. ``scripts/fetch_and_setup.py`` and LIC-FLOW-001 - (``licensing.py``) both rely on exactly this equality — LIC-FLOW-001 - passes each topic flowId straight to ``pp.get_flow(env_id, flow_id)`` - — so we do the same. - -Why the per-flow detail (not the listing): - The connection references block lives ONLY in the per-flow DETAIL - response. The flow LISTING (``pp.get_flows`` -> ``/v2/flows``) omits - ``properties.connectionReferences`` entirely (verified against the - cassette). Reading refs off the listing yields an empty allow-list, - which silently turns ENV-004 into a no-op — the bug this design avoids. - -External API contract tiers (per tests/fixtures/cassettes/INDEX.md): - - Dataverse ``botcomponents`` ``$select=name,schemaname,data`` / - ``$filter`` — ``documented`` tier (INDEX.md "API tier registry"; - the ``data`` column is the topic YAML per the MS Learn - ``botcomponent`` reference). ``$filter`` narrowing needs no cassette. - - Power Platform Admin ``/flows/{id}`` per-flow detail — ``validated`` - tier, cassette - ``tests/fixtures/cassettes/flightcheck_flow_licensing.yaml`` - (INDEX.md "Confirmed endpoints"); the detail record exposes - ``properties.connectionReferences..{connectionReferenceLogicalName,apiDefinition}``. - -Contract of :func:`build_agent_ref_scope`: - - Returns an :class:`AgentRefScope` when the agent's used refs were - resolved (possibly empty logical-name set if the matched flows carry - no connection references). - - Returns ``None`` when scoping cannot be established at all (no - configured botId, no Dataverse creds, no BAP client/env, no flowIds - discovered in the agent's topics, or none of the discovered flowIds - matched a flow returned by the admin surface). The caller SKIPs - rather than falling back to an env-wide verdict — a misleading - env-wide FAIL is worse than an honest SKIP. - - Raises on genuine API errors (Dataverse query exception, admin - ``_error`` payload). The caller converts these to a WARNING so the - failure is surfaced, not silently swallowed (design principle 3). -""" - -from __future__ import annotations - -import re -from dataclasses import dataclass - -from ._dlp_utils import normalize_connector_id -from auth import query_all # scripts/auth.py, on path via cli.py - - -# Matches ``flowId: `` (optionally quoted) inside an -# InvokeFlowAction block of a topic's YAML. Mirrors -# ``scripts/fetch_and_setup.py:discover_flow_ids_from_components`` so the -# two stay in agreement about what a topic flow reference looks like. -_FLOW_ID_RE = re.compile( - r'flowId:\s*["\']?([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-' - r'[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})', -) - - -@dataclass(frozen=True) -class AgentRefScope: - """The connection references (and their connectors) an agent uses. - - ``logical_names`` — lowercased ``connectionReferenceLogicalName`` - values the agent's flows bind to. ENV-004 keeps only Dataverse - ``connectionreference`` rows whose logical name is in this set. - - ``connectors`` — normalized connector ids (see - :func:`_dlp_utils.normalize_connector_id`) the agent's flows use. - ENV-004 scopes its unbound-connection (UC) warning to connections - whose connector is in this set, so unrelated apps' connections - don't get flagged. - """ - - logical_names: frozenset[str] - connectors: frozenset[str] - - -def _agent_bot_ids(config: dict) -> list[str]: - """Every configured agent botId (multi-agent + single-agent shapes).""" - bot_ids: list[str] = [] - for agent in config.get("agents", []) or []: - bid = (agent or {}).get("botId") - if bid: - bot_ids.append(bid) - if not bot_ids: - single = (config.get("agent") or {}).get("botId") - if single: - bot_ids.append(single) - # De-dupe while preserving order. - seen: set[str] = set() - ordered: list[str] = [] - for bid in bot_ids: - if bid not in seen: - seen.add(bid) - ordered.append(bid) - return ordered - - -def _extract_flow_ids(topic_data: str) -> set[str]: - """Lowercased flowIds referenced by InvokeFlowActions in a topic.""" - if not topic_data: - return set() - return {m.group(1).lower() for m in _FLOW_ID_RE.finditer(topic_data)} - - -def build_agent_ref_scope(runner) -> AgentRefScope | None: - """Resolve the connection references the configured agent(s) use. - - See the module docstring for the full contract. Returns an - :class:`AgentRefScope`, or ``None`` when scoping can't be - established. Raises on genuine API errors. - """ - config = getattr(runner, "config", None) or {} - bot_ids = _agent_bot_ids(config) - if not bot_ids: - return None - - env_url = getattr(runner, "env_url", None) - dv_token = getattr(runner, "dv_token", None) - if not env_url or not dv_token: - return None - - # --- Step 1: enabled topics -> flowIds --- - # statecode 0 = Active/Enabled (see local_files.py; a disabled topic - # never runs, so a ref only reachable through a disabled topic is not - # a live runtime dependency and must not drive a FAIL). - flow_ids: set[str] = set() - for bot_id in bot_ids: - topics = query_all( - env_url, dv_token, - "botcomponents", - "name,schemaname,data", - filter_expr=( - f"_parentbotid_value eq '{bot_id}' " - f"and componenttype eq 9 and statecode eq 0" - ), - ) - for topic in topics or []: - flow_ids |= _extract_flow_ids(topic.get("data") or "") - - if not flow_ids: - # The agent's enabled topics invoke no cloud flows we can see. - # We cannot build an allow-list, so we must not judge env-wide - # refs — signal "unresolvable" and let the caller SKIP. - return None - - # --- Step 2: read each agent flow's DETAIL for its connection refs --- - # A flow's connection references (and their connectors) live ONLY in - # the per-flow DETAIL response (``pp.get_flow`` -> ``/flows/{id}``). - # The flow LISTING (``pp.get_flows`` -> ``/v2/flows``) omits - # ``properties.connectionReferences`` entirely — verified against - # ``tests/fixtures/cassettes/flightcheck_flow_licensing.yaml`` (listing - # records carry only apiId/state/workflowEntityId/...). This mirrors - # LIC-FLOW-001 (``licensing.py``), which likewise resolves refs by - # calling ``get_flow`` per discovered flowId — the topic flowId is the - # flow's workflow GUID and the detail endpoint is keyed by it. - pp = getattr(runner, "pp_admin", None) - env_id = getattr(runner, "env_id", None) - if not pp or not env_id: - return None - - logical_names: set[str] = set() - connectors: set[str] = set() - readable = 0 - for flow_id in sorted(flow_ids): - try: - detail = pp.get_flow(env_id, flow_id) - except Exception as e: # surfaced as a WARNING by the caller - raise RuntimeError( - f"Power Platform Admin flow detail fetch failed for {flow_id}: {e}" - ) - # pp_admin maps 401/403 to {"_error","_status"} without raising. An - # auth/permission failure means scoping is unreliable, so fail - # loudly (principle 3) — the caller turns this into a WARNING. - if isinstance(detail, dict) and detail.get("_status") in (401, 403): - raise RuntimeError( - f"Power Platform Admin flow detail unauthorized for {flow_id}: " - f"{detail.get('_error')}" - ) - # A flow that is missing / not visible (404, None, other _error) - # just isn't readable; other flows may still resolve, so skip it - # rather than aborting the whole scope. - if not detail or (isinstance(detail, dict) and detail.get("_error")): - continue - readable += 1 - conn_refs = (detail.get("properties") or {}).get("connectionReferences") or {} - for _connector_key, meta in conn_refs.items(): - meta = meta or {} - logical = meta.get("connectionReferenceLogicalName") - if logical: - logical_names.add(str(logical).lower()) - api_def = meta.get("apiDefinition") or {} - connector = normalize_connector_id( - api_def.get("name") - or api_def.get("id") - or meta.get("apiName") - or meta.get("apiId") - ) - if connector: - connectors.add(connector) - - if readable == 0: - # We found flowIds in the agent's topics but none resolved to a - # readable flow on the admin surface (not visible to this caller, - # or genuinely absent). Scoping is unreliable — SKIP rather than - # under-report (a misleading env-wide verdict is worse). - return None - - return AgentRefScope( - logical_names=frozenset(logical_names), - connectors=frozenset(connectors), - ) diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/environment.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/environment.py index cfce8d6a0..f6cd40bcf 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/environment.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/environment.py @@ -10,14 +10,12 @@ import uuid from ..runner import CheckResult, Priority, Role, Status -from ._agent_connection_refs import build_agent_ref_scope -from ._dlp_utils import iter_effective_policies, normalize_connector_id -from ._maker_urls import ( - maker_connections_url, - maker_solution_url, - maker_solutions_url, +from ._da_connection_refs import ( + agent_bot_ids, + read_all_agents_connection_references, ) -from .connections import get_connection_status +from ._dlp_utils import iter_effective_policies +from ._maker_urls import maker_solutions_url from .licensing import ( _CAPACITY_DOC, _CAPACITY_PORTAL, @@ -29,186 +27,200 @@ DOC_BASE = "https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service" -def _resolve_ref_solutions( - *, env_url: str, dv_token: str, env_id: str, refs: list[dict], -) -> dict[str, dict[str, str]]: - """Resolve the containing solutions for a set of connection refs. - - Returns a map ``{connectionreferenceid: {"url": , "label": }}`` - so each ENV-004 detail row can deep-link to the specific solution - that holds its broken ref, instead of dumping the operator on the - env-wide solutions list. - - Two Dataverse round-trips are required because the - ``connectionreference`` entity carries no solution column — solution - membership is only exposed via the ``solutioncomponent`` intersect: - 1. ``solutioncomponents`` filtered only by ``objectid eq `` - → maps each ref GUID to its owning ``_solutionid_value``. We - deliberately omit a ``componenttype`` filter because Microsoft's - published enum does not document a stable value for Connection - Reference (earlier guesses such as 10047 returned empty in real - environments). A GUID is universally unique, so filtering on - ``objectid`` alone cannot collide with another component kind. - 2. ``solutions`` filtered by the distinct solution GUIDs from - step 1 *and* ``uniquename eq 'Default'`` → resolves - friendlyname/ismanaged for ranking, and ensures we always know - the Default Solution's GUID for the managed-only fallback. - - Link selection prefers (in order): a named unmanaged solution - containing the ref → the Default Solution (always unmanaged, always - present, can edit any component as customization) → no link. We - never link to a managed solution because Power Apps blocks edits - with "You cannot directly edit the objects within a managed - solution.", which is a dead end for the maker. - - Best-effort: any Dataverse failure returns ``{}`` so callers cleanly - fall back to the env-wide solutions URL. - """ - ref_ids = { - ref.get("connectionreferenceid") for ref in refs - if ref.get("connectionreferenceid") - } - if not ref_ids: - return {} +_ENV004_GRS_DESCRIPTION = "ESS agent GRS commit pin" +_ENV004_GRS_EXPECTED_COMMIT_KEYS = ( + "expectedGrsCommitSha", + "grsExpectedCommitSha", + "expectedMinimalBotsCommitSha", + "expectedEssSolutionCommitSha", +) +_ENV004_GRS_REALM_KEYS = ("minimalBotsAlmRealm", "grsRealm", "almRealm") +_ENV004_GRS_DEFAULT_REALM = "Dev" +# Config realm string -> the numeric realm the AgentBuilder ALM configure API +# expects (mirrors agentbuilder.REALM_NAMES: 0=Dev, 1=Test, 2=Prod). +_ENV004_GRS_REALMS = {"dev": 0, "test": 1, "prod": 2, "production": 2} +_ENV004_REALM_DISPLAY = {0: "Dev", 1: "Test", 2: "Prod"} - # Step 1: ref-guid -> solution-guid via solutioncomponents. - # No componenttype filter — see docstring above for rationale. - sc_filter = " or ".join(f"objectid eq {rid}" for rid in ref_ids) - try: - components = query_all( - env_url, dv_token, - "solutioncomponents", - "objectid,_solutionid_value", - filter_expr=sc_filter, - ) - except Exception: - return {} - # A single ref typically appears in multiple solution layers (the - # base managed solution that defined it plus any unmanaged layer - # that customized it). Collect every solution per ref so step 2 can - # pick the one the maker can actually edit in the portal. - ref_to_solutions: dict[str, list[str]] = {} - for comp in components: - oid = comp.get("objectid") - sid = comp.get("_solutionid_value") - if oid and sid: - ref_to_solutions.setdefault(oid, []).append(sid) - if not ref_to_solutions: +def _env004_active_agent_config(config: dict) -> dict: + """Return the active agent block from the setup config, if present.""" + if not isinstance(config, dict): return {} + agent = config.get("agent") + if isinstance(agent, dict) and agent: + return agent + + agents = config.get("agents") or [] + active = config.get("activeAgent", "") + if isinstance(agents, list): + for candidate in agents: + if isinstance(candidate, dict) and candidate.get("slug") == active: + return candidate + for candidate in agents: + if isinstance(candidate, dict): + return candidate + return {} + + +def _env004_config_value(config: dict, keys: tuple[str, ...]) -> str: + """Read a string setting from active-agent config first, then top-level.""" + active_agent = _env004_active_agent_config(config) + for source in (active_agent, config): + if not isinstance(source, dict): + continue + for key in keys: + value = source.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return "" + + +def _env004_grs_commit_pin_result(runner) -> CheckResult: + """Validate the minimalBots ALM commit pin (ENV-004-GRS). + + Opt-in component-layer check: verifies the deployed Declarative Agent's ALM + commit matches an expected GRS commit SHA when one is recorded in config. + Validated-tier read (GET .../alm/{id}/configure?realm={int} -> ``commitSha``; + cassette ``agentbuilder_readiness.yaml``). SKIPs when no expected SHA is + configured (the pin is inert until an operator records the intended release + commit), so this never blocks a run that does not use pinning. + """ + config = getattr(runner, "config", None) or {} + expected_commit = _env004_config_value( + config, _ENV004_GRS_EXPECTED_COMMIT_KEYS + ) + realm_raw = ( + _env004_config_value(config, _ENV004_GRS_REALM_KEYS) + or _ENV004_GRS_DEFAULT_REALM + ) + # 0 (Dev) is a valid realm and falsy, so test membership explicitly. + realm = _ENV004_GRS_REALMS.get(realm_raw.lower()) + + if not expected_commit: + return CheckResult( + roles=[Role.ESS_MAKER.value], + checkpoint_id="ENV-004-GRS", + category="Environment", + priority=Priority.HIGH.value, + status=Status.SKIPPED.value, + description=_ENV004_GRS_DESCRIPTION, + result=( + "No expected GRS commit SHA is configured, so the minimalBots " + "ALM commit pin was not judged." + ), + remediation=( + "Record the expected ESS solution commit SHA in " + ".local/config.json using expectedGrsCommitSha (top-level or on " + "the active agent), then re-run FlightCheck." + ), + ) - # Step 2: solution-guid -> {friendlyname, ismanaged} via solutions. - # We also unconditionally include the **Default Solution** (uniquename - # 'Default'), which is the unmanaged customization layer that always - # exists in every Dataverse environment. It's the only place a maker - # can edit a connection reference that was defined in a managed - # solution — Power Apps refuses direct edits there with "You cannot - # directly edit the objects within a managed solution." So when a ref - # only lives in managed solutions, we fall back to Default Solution. - distinct_sids = {sid for sids in ref_to_solutions.values() for sid in sids} - sid_clauses = [f"solutionid eq {sid}" for sid in distinct_sids] - sol_filter = "(" + " or ".join(sid_clauses) + ") or uniquename eq 'Default'" - try: - solutions = query_all( - env_url, dv_token, - "solutions", - "solutionid,uniquename,friendlyname,ismanaged", - filter_expr=sol_filter, + if realm is None: + return CheckResult( + roles=[Role.ESS_MAKER.value], + checkpoint_id="ENV-004-GRS", + category="Environment", + priority=Priority.HIGH.value, + status=Status.FAILED.value, + description=_ENV004_GRS_DESCRIPTION, + result=( + f"Configured minimalBots ALM realm '{realm_raw}' is invalid. " + "Expected one of Dev, Test, or Prod." + ), + remediation=( + "Set minimalBotsAlmRealm, grsRealm, or almRealm in " + ".local/config.json to Dev, Test, or Prod." + ), ) - except Exception: - return {} - sid_to_info: dict[str, dict] = {} - default_sid: str | None = None - for sol in solutions: - sid = sol.get("solutionid") - if not sid: - continue - uname = sol.get("uniquename") or "" - sid_to_info[sid] = { - "label": ( - sol.get("friendlyname") or uname or sid + realm_name = _ENV004_REALM_DISPLAY[realm] + client = getattr(runner, "agentbuilder", None) + bot_ids = agent_bot_ids(config) + if client is None or not bot_ids: + return CheckResult( + roles=[Role.ESS_MAKER.value], + checkpoint_id="ENV-004-GRS", + category="Environment", + priority=Priority.HIGH.value, + status=Status.SKIPPED.value, + description=_ENV004_GRS_DESCRIPTION, + result=( + "AgentBuilder client or agent botId not available, so the " + "minimalBots ALM commit pin was not judged." + ), + remediation=( + "Run /setup so .local/config.json records the agent botId, and " + "ensure FlightCheck is signed in to Copilot Studio (AgentBuilder), " + "then re-run FlightCheck." ), - "uniquename": uname, - "ismanaged": bool(sol.get("ismanaged")), - } - if uname == "Default": - default_sid = sid - - # Internal/system solutions a maker never opens in the portal — skip - # them when ranking. (Note: 'Default' is the user-facing Default - # Solution and IS editable, so it's NOT in this set.) - SYSTEM_UNIQUENAMES = {"Active", "Basic", "System"} - - def _solution_rank(sid: str) -> tuple[int, int, int, str]: - info = sid_to_info.get(sid, {}) - uname = info.get("uniquename", "") - is_system = uname in SYSTEM_UNIQUENAMES - is_managed = info.get("ismanaged", False) - is_default = uname == "Default" - # Lowest tuple wins. Prefer (in order): - # 1. not-system over system, - # 2. unmanaged over managed (maker can edit directly), - # 3. a named unmanaged solution over Default (more focused - # view; Default is the catch-all), - # 4. alphabetical for stability. - return ( - 1 if is_system else 0, - 1 if is_managed else 0, - 1 if is_default else 0, - uname, ) - out: dict[str, dict[str, str]] = {} - for rid, sids in ref_to_solutions.items(): - # Only consider sids we successfully resolved in step 2. - known = [sid for sid in sids if sid in sid_to_info] - if known: - best = sorted(known, key=_solution_rank)[0] - # If the best candidate is still managed (i.e. every solution - # containing this ref is managed), redirect the link to the - # Default Solution so the maker actually has somewhere to - # edit. The label changes too so the report doesn't mislead - # the maker into clicking through to a read-only solution. - if sid_to_info[best].get("ismanaged"): - if default_sid: - out[rid] = { - "url": maker_solution_url(env_id, default_sid), - "label": sid_to_info[default_sid]["label"], - } - # Else: nothing editable to link to. Skip so the caller - # falls back to the env-wide URL — better to give the - # maker the solutions list than dump them on a managed - # solution page that refuses edits. - continue - out[rid] = { - "url": maker_solution_url(env_id, best), - "label": sid_to_info[best]["label"], - } - elif default_sid: - # No containing solution resolved — Default Solution is still - # the right editable fallback (better than the env-wide list). - out[rid] = { - "url": maker_solution_url(env_id, default_sid), - "label": sid_to_info[default_sid]["label"], - } - return out - - -def _solution_link_parts( - ref: dict, solution_info: dict[str, dict[str, str]], fallback_url: str, -) -> tuple[str, str]: - """Pick the best (url, label) for the solution containing a ref. - - Returns a deep link to the specific solution when we resolved its - metadata; otherwise falls back to the env-wide solutions list so - the remediation never produces a 404. - """ - rid = ref.get("connectionreferenceid") - if rid and rid in solution_info: - info = solution_info[rid] - return info["url"], f"Power Apps \u2192 Solutions \u2192 {info['label']}" - return fallback_url, "Power Apps \u2192 Solutions" + mismatches: list[tuple[str, str]] = [] + missing_commit: list[str] = [] + for bot_id in bot_ids: + try: + data = client.get_realm_configuration(bot_id, realm) + except Exception as e: # noqa: BLE001 — surface a read failure as WARNING + return CheckResult( + roles=[Role.ESS_MAKER.value], + checkpoint_id="ENV-004-GRS", + category="Environment", + priority=Priority.HIGH.value, + status=Status.WARNING.value, + description=_ENV004_GRS_DESCRIPTION, + result=( + f"Could not read minimalBots ALM configure for realm " + f"{realm_name}: {type(e).__name__}: {e}" + ), + remediation=( + "Ensure the agent is opted into minimalBots ALM for this realm " + "and that FlightCheck is signed in to Copilot Studio (AgentBuilder)." + ), + ) + observed_commit = (data or {}).get("commitSha") or "" + if not observed_commit: + missing_commit.append(bot_id) + elif observed_commit.lower() != expected_commit.lower(): + mismatches.append((bot_id, observed_commit)) + + if mismatches or missing_commit: + parts = [ + f"botId {bot_id} has commitSha {observed}" + for bot_id, observed in mismatches + ] + parts.extend( + f"botId {bot_id} returned no commitSha" for bot_id in missing_commit + ) + return CheckResult( + roles=[Role.ESS_MAKER.value], + checkpoint_id="ENV-004-GRS", + category="Environment", + priority=Priority.HIGH.value, + status=Status.FAILED.value, + description=_ENV004_GRS_DESCRIPTION, + result=( + f"Expected GRS commit SHA {expected_commit} for realm " + f"{realm_name}, but " + "; ".join(parts) + ), + remediation=( + "Publish or import the ESS agent solution built from the expected " + "commit, or update expectedGrsCommitSha only after confirming the " + "new commit is the intended release." + ), + ) + + return CheckResult( + roles=[Role.ESS_MAKER.value], + checkpoint_id="ENV-004-GRS", + category="Environment", + priority=Priority.HIGH.value, + status=Status.PASSED.value, + description=_ENV004_GRS_DESCRIPTION, + result=( + f"minimalBots ALM configure for realm {realm_name} reports the " + f"expected GRS commit SHA {expected_commit}." + ), + ) def run_environment_checks(runner) -> list[CheckResult]: @@ -445,462 +457,158 @@ def _check_copilot_studio_capacity_provisioned(runner) -> list[CheckResult]: # --------------------------------------------------------------------------- def _check_connections_and_refs(runner) -> list[CheckResult]: - """Report the agent's connection references with binding state. - - Scope: only the connection references the ESS agent(s) under check - actually use are judged — resolved from each agent's enabled topics - -> InvokeFlowAction flowIds -> BAP flow ``connectionReferences`` - (see ``_agent_connection_refs.build_agent_ref_scope``). References - belonging to other apps in the same environment, and the ESS-shipped - placeholder references that ship unbound-by-design on a Workday - simplified install, are excluded so they don't produce false FAILs. - When the agent's ref set can't be resolved, the check SKIPs rather - than judging environment-wide references. - - Detects broken bindings in both directions: - - Connection references pointing to a connection that doesn't exist - (orphan reference). - - Connection references the agent's flows expect that have no row in - the environment (missing reference). - - Connections (of a connector the agent uses) with no corresponding - in-scope connection reference (unbound connection). - - Terminology: - **Orphan reference** (FAIL) — An in-scope connection reference points - to a connection ID that no longer exists in the environment. This - occurs when a connection was deleted, the solution was imported from - another environment, or a connection was recreated with a new ID. - Topics/flows using this reference will fail at runtime with auth errors. - - **Unbound reference** (FAIL) — An in-scope connection reference exists - but has no connection ID set (empty ``connectionid`` field). This - occurs after a solution import where references were never configured, - or a new reference was added but not bound. Topics/flows using this - reference will fail immediately. - - **Missing reference** (FAIL) — The agent's flow references a connection - reference logical name that has no ``connectionreference`` row in this - environment at all. The flow fails at runtime with an unresolved - reference; re-importing the agent's solution recreates it. - - **Unbound connection** (WARN) — A connection whose connector the agent - uses exists in the environment but no in-scope connection reference - points to it. Common after troubleshooting (test connections), - re-binding references to newer connections, or manual connection - creation. No runtime impact, but adds clutter. + """Report the Declarative Agent's connection references, plus the GRS pin. + + Re-pointed from the Dataverse ``connectionreference`` table to the + Declarative Agent minimalBots components API (validated tier; the same + ``connectionReferenceChanges`` shape the shipped native ``DA-CONN-001`` + check consumes, read via ``_da_connection_refs``). The DA components + changeset IS each agent's declared reference set, so this reads every + configured agent's references and classifies each: + + - **Unbound reference** (FAIL) \u2014 a reference with no ``connectionId``. + Topics/flows using it fail immediately at runtime. + - **Bound reference** (PASS) \u2014 a reference with a ``connectionId`` set. + + Unlike the former Dataverse-backed check, the DA changeset carries no + environment-wide connection inventory, so the orphan-reference, + missing-reference, and unbound-connection branches are not applicable and + are not emitted. + + A separate GRS commit-pin sub-check (``ENV-004-GRS``) verifies the deployed + agent's ALM commit matches an expected SHA when one is configured; its + verdict folds into the ENV-004 summary status and it is also emitted as a + detail row (so a scope run shows it; a targeted ``--checkpoint ENV-004`` run + filters the detail row out but still sees the folded verdict in the summary). Signal: - SKIP — the agent's connection-reference set could not be resolved. - PASS — all in-scope references bound to existing connections. - WARN — unbound connections of an agent connector exist (may be intentional). - FAIL — orphan, unbound, or missing references found (broken bindings). + SKIP \u2014 no AgentBuilder client or no configured agent botId. + PASS \u2014 every reference is bound (and the GRS pin passed or was not judged). + WARN \u2014 the components read or GRS configure read errored. + FAIL \u2014 one or more unbound references (or the GRS pin failed). """ - results: list[CheckResult] = [] - pp = runner.pp_admin - env_id = runner.env_id - env_url = getattr(runner, "env_url", None) - dv_token = getattr(runner, "dv_token", None) - - if not pp or not env_id: - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id="ENV-004", category="Environment", - priority=Priority.HIGH.value, status=Status.SKIPPED.value, - description="Connections & connection references", - result="Power Platform Admin API not available — skipping", - remediation="Requires Power Platform Administrator role.", - )) - return results - - if not env_url or not dv_token: - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id="ENV-004", category="Environment", - priority=Priority.HIGH.value, status=Status.SKIPPED.value, - description="Connections & connection references", - result="Dataverse token not available — cannot query connection references", - remediation="Ensure Dataverse authentication is configured.", - )) - return results - - # --- Fetch connections from PP Admin API --- - try: - all_conns = pp.get_connections(env_id) - if isinstance(all_conns, dict) and "_error" in all_conns: - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id="ENV-004", category="Environment", - priority=Priority.HIGH.value, status=Status.WARNING.value, - description="Connections & connection references", - result=f"Unable to list connections: {all_conns['_error']}", - remediation="Requires Power Platform Admin role.", - )) - return results - except Exception as e: - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id="ENV-004", category="Environment", - priority=Priority.HIGH.value, status=Status.WARNING.value, - description="Connections & connection references", - result=f"Error fetching connections: {e}", - )) - return results - - # --- Fetch connection references from Dataverse --- - # - # `connectionreference` carries no solution column — solution - # membership is only exposed via the `solutioncomponent` intersect. - # The deep-link resolution (`_resolve_ref_solutions`) does the - # extra round-trip downstream, only for the broken refs we need - # to remediate, not for every ref in the env. - try: - conn_refs = query_all( - env_url, dv_token, - "connectionreferences", - "connectionreferenceid,connectionreferencelogicalname," - "connectorid,connectionid,connectionreferencedisplayname,statuscode", - ) - except Exception as e: - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id="ENV-004", category="Environment", - priority=Priority.HIGH.value, status=Status.WARNING.value, - description="Connections & connection references", - result=f"Error querying connection references: {e}", - remediation="Ensure Dataverse access permissions.", - )) - return results + roles = [Role.POWER_PLATFORM_ADMIN.value] + description = "Connections & connection references" - # --- Scope to the connection references THIS agent actually uses --- - # - # Judging every ref in the environment produces false FAILs on refs - # the ESS agent never touches (other apps' refs, and the ESS-shipped - # placeholder refs that ship unbound-by-design on a Workday - # simplified install). `build_agent_ref_scope` resolves the agent's - # used refs from its enabled topics -> InvokeFlowAction flowIds -> - # BAP flow connectionReferences. See _agent_connection_refs.py. - # - # - None -> we can't establish the agent's ref set. SKIP rather - # than judge env-wide (a misleading FAIL is worse than - # an honest SKIP). - # - raise -> genuine API error; surface as WARNING (principle 3). - # - # Catch only RuntimeError: that's the exception type the builder - # raises for genuine API failures (see build_agent_ref_scope's - # contract). Letting any other exception type propagate means a - # future programming defect surfaces loudly as an ERROR (via the - # runner) instead of being disguised as a benign WARNING here. + # Validated-tier read via the shared DA reader. Fail loudly: a malformed + # changeset raises (degrade to WARNING) rather than reporting a confident + # but wrong verdict; an unavailable client/botId returns None (SKIP). try: - ref_scope = build_agent_ref_scope(runner) - except RuntimeError as e: - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], + refs = read_all_agents_connection_references(runner) + except Exception as e: # noqa: BLE001 \u2014 fail loudly as a WARNING + results.append(CheckResult(roles=roles, checkpoint_id="ENV-004", category="Environment", priority=Priority.HIGH.value, status=Status.WARNING.value, - description="Connections & connection references", - result=f"Unable to determine which connection references this agent uses: {e}", + description=description, + result=( + f"Unable to read the agent's connection references: " + f"{type(e).__name__}: {e}" + ), remediation=( - "Re-run FlightCheck with Power Platform Administrator access so the " - "agent's cloud flows can be listed and its connection references scoped." + "Re-run FlightCheck; if this persists, ensure FlightCheck is " + "signed in to Copilot Studio (AgentBuilder) and report the error above." ), )) return results - if ref_scope is None: - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], + if refs is None: + results.append(CheckResult(roles=roles, checkpoint_id="ENV-004", category="Environment", priority=Priority.HIGH.value, status=Status.SKIPPED.value, - description="Connections & connection references", + description=description, result=( - "Could not determine which connection references belong to this agent " - "(no configured agent botId, or the agent's cloud flows could not be " - "resolved), so environment-wide references were not judged." + "AgentBuilder client or agent botId not available, so connection " + "references were not judged." ), remediation=( - "Ensure .local/config.json carries the agent's botId and that FlightCheck " - "is signed in with Power Platform Administrator access, then re-run so the " - "agent's flows (and their connection references) can be enumerated." + "Run /setup so .local/config.json records the agent botId, and " + "ensure FlightCheck is signed in to Copilot Studio (AgentBuilder), " + "then re-run FlightCheck." ), )) return results - # Keep only the Dataverse refs whose logical name is one the agent's - # flows bind to. Presence of the full env-wide set is retained - # separately so we can still detect references the flows expect but - # that don't exist in the environment at all (missing refs, below). - all_ref_logical_names = { - (r.get("connectionreferencelogicalname") or "").lower() - for r in conn_refs - } - in_scope_refs = [ - r for r in conn_refs - if (r.get("connectionreferencelogicalname") or "").lower() - in ref_scope.logical_names - ] - - # Missing references: a logical name the agent's flows reference but - # that has no `connectionreference` row anywhere in the environment. - # The flow will fail at runtime with an unresolved-reference error. - missing_ref_names = sorted( - name for name in ref_scope.logical_names - if name and name not in all_ref_logical_names - ) - - # --- Build lookup: connection name (GUID) → connection object --- - conn_map = {} - for c in all_conns: - conn_name = c.get("name", "") - if conn_name: - conn_map[conn_name] = c - - # --- Analyze binding state (agent-scoped refs only) --- - bound_conn_ids = set() - orphan_refs = [] # References pointing to non-existent connections - unbound_refs = [] # References with no connectionid set - - for ref in in_scope_refs: - conn_id = ref.get("connectionid") or "" - if not conn_id: - unbound_refs.append(ref) - elif conn_id in conn_map: - bound_conn_ids.add(conn_id) - else: - orphan_refs.append(ref) - - # Unbound connections: connections no in-scope reference points to. - # Scope this to connectors the agent actually uses so unrelated - # apps' connections in the same environment aren't flagged. - unbound_conns = [ - c for c in all_conns - if c.get("name", "") - and c.get("name", "") not in bound_conn_ids - and normalize_connector_id( - (c.get("properties", {}) or {}).get("apiId", "") - ) in ref_scope.connectors - ] + unbound_refs = [r for r in refs if not (r.get("connectionid") or "")] + bound_refs = [r for r in refs if (r.get("connectionid") or "")] - # --- Determine overall status --- - has_failing_refs = ( - len(orphan_refs) > 0 or len(unbound_refs) > 0 or len(missing_ref_names) > 0 - ) - has_unbound_conns = len(unbound_conns) > 0 + # The GRS commit pin is a distinct ENV-004 sub-check; fold its verdict into + # the summary status so a targeted run (which filters the detail row) still + # surfaces a GRS failure. + grs_result = _env004_grs_commit_pin_result(runner) - if has_failing_refs: + if unbound_refs or grs_result.status == Status.FAILED.value: overall_status = Status.FAILED.value - elif has_unbound_conns: + elif grs_result.status == Status.WARNING.value: overall_status = Status.WARNING.value else: overall_status = Status.PASSED.value - # --- Summary (agent-scoped) --- - bound_refs = len(in_scope_refs) - len(orphan_refs) - len(unbound_refs) summary_parts = [ - f"{len(all_conns)} connection(s) in environment", - f"{len(in_scope_refs)} reference(s) used by this agent", - f"{bound_refs} bound ({len(bound_conn_ids)} distinct conn(s))", + f"{len(refs)} reference(s) declared by the agent(s)", + f"{len(bound_refs)} bound", ] - if orphan_refs: - summary_parts.append(f"{len(orphan_refs)} orphan ref(s)") if unbound_refs: - summary_parts.append(f"{len(unbound_refs)} unbound ref(s)") - if missing_ref_names: - summary_parts.append(f"{len(missing_ref_names)} missing ref(s)") - if unbound_conns: - summary_parts.append(f"{len(unbound_conns)} unbound conn(s)") - - remediation = "" - solutions_url = maker_solutions_url(env_id) - connections_url = maker_connections_url(env_id) - # Microsoft's official walkthrough for binding / editing a - # connection reference. We surface this as doc_link so the - # operator can follow the canonical flow if the abbreviated - # in-remediation instructions aren't enough. + summary_parts.append(f"{len(unbound_refs)} unbound") + if grs_result.status != Status.SKIPPED.value: + summary_parts.append(f"GRS commit pin: {grs_result.status}") + + env_id = getattr(runner, "env_id", None) + solutions_url = maker_solutions_url(env_id) if env_id else None conn_ref_doc = ( "https://learn.microsoft.com/en-us/power-apps/maker/" "data-platform/create-connection-reference" ) - # --- Resolve containing solution for each problematic ref --- - # - # The env-wide solutions list dumps every first-party + ISV solution - # in the env on the operator and leaves them guessing which one - # holds the broken ref. Query the `solutioncomponent` intersect - # (componenttype 10047 = Connection Reference) to map each broken - # ref's GUID to its owning solution, then look up the friendly - # display name + solution GUID so the per-row remediation can - # deep-link straight to the right solution's detail page. - # - # The lookup is best-effort: any failure (Dataverse error, missing - # field, missing solution) cleanly falls back to the env-wide - # solutions list URL so the remediation never silently 404s. - problematic_refs = orphan_refs + unbound_refs - solution_info = _resolve_ref_solutions( - env_url=env_url, dv_token=dv_token, - env_id=env_id, refs=problematic_refs, - ) - - if has_failing_refs: - # Build the most specific summary remediation we can: - # - All broken refs in ONE resolved solution → deep-link to it - # - Broken refs span MULTIPLE resolved solutions → name them - # all, but the link has to fall back to the env-wide list - # (no single deep link covers multiple solutions) - # - Lookup didn't resolve any solution → generic prose - distinct_solutions = { - (info["url"], info["label"]) - for info in solution_info.values() - } - if len(distinct_solutions) == 1: - sol_url, sol_label = next(iter(distinct_solutions)) - remediation = ( - f"Fix broken connection references: open [Power Apps \u2192 Solutions " - f"\u2192 {sol_label}]({sol_url}) \u2192 in the left nav choose " - f"**Objects \u2192 Connection references** \u2192 re-bind each broken " - f"reference to a valid connection, or remove stale references." - ) - elif len(distinct_solutions) > 1: - names = ", ".join(sorted(label for _, label in distinct_solutions)) - remediation = ( - f"Fix broken connection references (spread across solutions: {names}). " - f"Open [Power Apps \u2192 Solutions]({solutions_url}), open each " - f"affected solution, and in the left nav choose **Objects \u2192 " - f"Connection references** to re-bind each broken reference or remove " - f"stale ones. See the ENV-004-OR-* / ENV-004-UR-* detail rows below " - f"for per-reference deep links." - ) - else: - remediation = ( - f"Fix broken connection references: open [Power Apps \u2192 Solutions]({solutions_url}) " - f"\u2192 click the solution that contains your agent \u2192 in the left nav choose " - f"**Objects \u2192 Connection references** \u2192 re-bind each broken reference to a " - f"valid connection, or remove stale references." - ) - elif has_unbound_conns: + if unbound_refs and solutions_url: + remediation = ( + f"Bind the unbound connection reference(s): open [Power Apps \u2192 " + f"Solutions]({solutions_url}) \u2192 open the solution that contains " + f"your agent \u2192 in the left nav choose **Objects \u2192 Connection " + f"references** \u2192 bind each unbound reference to a valid connection." + ) + elif unbound_refs: remediation = ( - f"Unbound connections may be intentional (e.g. test connections). " - f"Review them in [the environment connections list]({connections_url}) and remove unused entries." + "Bind the unbound connection reference(s) in Power Apps \u2192 " + "Solutions \u2192 your agent's solution \u2192 Objects \u2192 " + "Connection references." ) + else: + remediation = "" - # When the summary is FAILED, the operator needs to fix connection - # references; surface Microsoft's canonical walkthrough as doc_link - # so they have the full reference next to the abbreviated steps. summary_doc_link = ( conn_ref_doc if overall_status == Status.FAILED.value else f"{DOC_BASE}/prepare#set-up-your-power-platform-environment" ) - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], + results.append(CheckResult(roles=roles, checkpoint_id="ENV-004", category="Environment", priority=Priority.HIGH.value, status=overall_status, - description="Connections & connection references", + description=description, result=" | ".join(summary_parts), remediation=remediation, doc_link=summary_doc_link, )) - # --- Detail: orphan references (point to missing connections) --- - for i, ref in enumerate(orphan_refs): - ref_name = ref.get("connectionreferencedisplayname") or ref.get( - "connectionreferencelogicalname", "Unknown" - ) - dead_conn_id = ref.get("connectionid", "?") - sol_url, sol_label = _solution_link_parts(ref, solution_info, solutions_url) - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id=f"ENV-004-OR-{i + 1:03d}", category="Environment", - priority=Priority.HIGH.value, status=Status.FAILED.value, - description=f"Orphan reference: {ref_name}", - result=f"Points to missing connection '{dead_conn_id}'", - remediation=( - f"Open [{sol_label}]({sol_url}) \u2192 in the left nav choose **Objects " - f"\u2192 Connection references** \u2192 re-bind '{ref_name}' to an active " - f"connection, or delete the reference." - ), - doc_link=conn_ref_doc, - )) - - # --- Detail: unbound references (no connectionid set) --- + # --- Detail: unbound references (no connectionId set) --- for i, ref in enumerate(unbound_refs): - ref_name = ref.get("connectionreferencedisplayname") or ref.get( - "connectionreferencelogicalname", "Unknown" - ) - sol_url, sol_label = _solution_link_parts(ref, solution_info, solutions_url) - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], + ref_name = ref.get("connectionreferencelogicalname") or "Unknown" + connector = ref.get("connectorid") or "?" + results.append(CheckResult(roles=roles, checkpoint_id=f"ENV-004-UR-{i + 1:03d}", category="Environment", priority=Priority.HIGH.value, status=Status.FAILED.value, description=f"Unbound reference: {ref_name}", - result="No connection bound to this reference", + result=f"No connection bound to this reference (connector {connector})", remediation=( - f"Open [{sol_label}]({sol_url}) \u2192 in the left nav choose **Objects " - f"\u2192 Connection references** \u2192 bind '{ref_name}' to a valid connection." + f"Open your agent's solution in Power Apps \u2192 Solutions \u2192 " + f"**Objects \u2192 Connection references** \u2192 bind '{ref_name}' to " + f"a valid connection." ), doc_link=conn_ref_doc, )) - # --- Detail: missing references (agent flow references a connection - # reference logical name that has no row in this environment) --- - # The flow expects this reference to exist; without it the flow fails - # at runtime with an unresolved-connection-reference error. Unlike an - # unbound ref (row exists, no connection bound), here the row itself - # is absent — importing/re-deploying the agent's solution recreates it. - for i, logical_name in enumerate(missing_ref_names): - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id=f"ENV-004-MR-{i + 1:03d}", category="Environment", - priority=Priority.HIGH.value, status=Status.FAILED.value, - description=f"Missing reference: {logical_name}", - result=( - f"An agent cloud flow references connection reference " - f"'{logical_name}', but no such reference exists in this environment" - ), - remediation=( - f"The agent's flow expects a connection reference named " - f"'{logical_name}' that is not present in this environment, so the flow " - f"fails at runtime with an unresolved-reference error. Re-import (or push) " - f"the agent's solution so the missing reference is recreated, then open " - f"[Power Apps \u2192 Solutions]({solutions_url}) \u2192 **Objects \u2192 " - f"Connection references** and bind it to a valid connection." - ), - doc_link=conn_ref_doc, - )) - - # --- Detail: unbound connections (no reference in THIS agent's solution - # points to them). ``unbound`` here is scoped to the agent under check: - # the connection might still be in use by another agent, a Power Automate - # flow that connects directly without a connection reference, or a - # canvas/model-driven app. We don't query env-wide to confirm true - # disuse, so the remediation MUST be explicit about that limitation - # and walk the maker through verifying before deletion. Deleting a - # connection that something else depends on breaks that resource - # silently \u2014 the platform does not warn. - for i, conn in enumerate(unbound_conns): - props = conn.get("properties", {}) - conn_name = props.get("displayName", conn.get("name", "Unknown")) - api_id = props.get("apiId", "") - connector_label = api_id.split("/")[-1] if api_id else "unknown" - conn_status = get_connection_status(conn) - results.append(CheckResult(roles=[Role.POWER_PLATFORM_ADMIN.value], - checkpoint_id=f"ENV-004-UC-{i + 1:03d}", category="Environment", - priority=Priority.MEDIUM.value, status=Status.WARNING.value, - description=f"Unbound connection: {conn_name}", - result=( - f"Connector: {connector_label} | Status: {conn_status} | " - f"Not referenced by this agent's solution" - ), - remediation=( - f"This connection is not referenced by THIS agent's solution, " - f"but it may still be used by another agent, a Power Automate " - f"flow, or an app in this environment. **Verify it is unused " - f"before deleting** \u2014 the platform does not warn if you " - f"delete a connection that something else depends on. " - f"To verify: " - f"(1) open [Power Automate \u2192 Connections]({connections_url}) " - f"and click '{conn_name}' \u2014 the detail page lists apps " - f"that depend on it; " - f"(2) open [Power Automate \u2192 My flows]" - f"(https://make.powerautomate.com/environments/{runner.env_id}/flows) " - f"and check whether any flow authenticates via the " - f"'{connector_label}' connector; " - f"(3) open other [solutions in this environment]" - f"(https://make.powerapps.com/environments/{runner.env_id}/solutions) " - f"and check their **Objects \u2192 Connection references**. " - f"If nothing depends on '{conn_name}', delete it from the " - f"Power Automate Connections list." - ), - )) + # --- Detail: GRS commit pin (also folded into the summary status above) --- + results.append(grs_result) return results diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py index 12746ce56..b3a6b49ad 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py @@ -20,11 +20,12 @@ (never assert a verdict from an unconfirmed API response shape) — this checkpoint echoes the observed ``connectionParametersSet.name`` for the operator to confirm, rather than PASS/FAIL on a guessed value. - * ``DV-CONN-001`` (S5.4) — the Dataverse connection reference the extension - pack ships (``…_92b66``, connector ``shared_commondataserviceforapps``) is - bound to an **active** connection, and its owner is echoed so the operator - can confirm it is their **own** account. Programmatic PASS/FAIL on a - documented-tier Dataverse ``connectionreferences`` read. + * ``DV-CONN-001`` (S5.4) — the Workday SOAP connection reference reported by + the Declarative Agent minimalBots components API is bound (``connectionId`` + present), and its owner is echoed so the operator can confirm it is their + **own** account. Programmatic PASS/FAIL on the validated minimalBots + components read (same endpoint + ``connectionReferenceChanges`` shape as the + shipped native ``DA-CONN-001`` check). * ``WD-REST-001`` (S5.5) — the captured ``restBaseUrl`` is present and **trimmed to** ``/api``. Pure-config check, no client. * ``WD-REST-002`` (S5.7) — the agent's ``user-context-setup.mcs.yml`` topic @@ -43,7 +44,7 @@ whole run. * **One CheckResult per checkpoint** (principle 7). * **No guessed API shapes** — the two API-backed checks read documented fields - only (Dataverse ``connectionid`` / ``statuscode``; BAP + only (minimalBots ``connectionReferenceChanges`` connector/connection ids; BAP ``connectionParametersSet.name`` / ``createdBy``), and degrade gracefully when a client is unavailable. * **Every** ``CheckResult`` declares ``roles=`` (enforced by @@ -52,19 +53,13 @@ from __future__ import annotations -import os import re -import sys from pathlib import Path from ..runner import CheckResult, Priority, Role, Status +from ._da_connection_refs import read_active_agent_connection_references from ..agent_scope import resolve_agent_directory, validate_agent_slug -# scripts/auth.py is on sys.path via cli.py at runtime (tests add it too); this -# mirrors checks/environment.py's top-level import so query_all is patchable as -# flightcheck.checks.workday_extension.query_all. -from auth import query_all # noqa: E402 - DOC_BASE = ( "https://learn.microsoft.com/en-us/copilot/microsoft-365/" "employee-self-service" @@ -92,12 +87,9 @@ _WORKDAY_RUNTIME_REF_LOGICAL_NAME = ( "msdyn_sharedworkdaysoap_workdayruntime" ) -# The Dataverse connection reference the simplified pack ships. -_DATAVERSE_CONNECTOR_SUFFIX = "/apis/shared_commondataserviceforapps" -_DATAVERSE_REF_SUFFIX = "92b66" -_DATAVERSE_RUNTIME_REF_LOGICAL_NAME = ( - "msdyn_sharedcommondataserviceforapps_workdayruntime" -) +# The Workday SOAP connection reference the Declarative Agent reports via the +# minimalBots components API (connector ``shared_workdaysoap``). +_WORKDAY_CONNECTOR_SUFFIX = "/apis/shared_workdaysoap" _REF_SUFFIX_RE = re.compile(r"_([0-9a-f]{5})$") # ---- Local user-context topic (WD-REST-002) ---- @@ -110,7 +102,7 @@ "Workday connection authentication type is Microsoft Entra ID Integrated" ) _DV_CONN_DESC = ( - "Dataverse connection reference bound to an active connection you own" + "Workday SOAP connection reference bound to a connection you own" ) _REST_URL_DESC = "Workday REST base URL present and trimmed to '/api'" _REDIRECT_DESC = ( @@ -194,14 +186,6 @@ def _is_workday_auth_ref(logical_name) -> bool: ) -def _is_dataverse_runtime_ref(logical_name) -> bool: - normalized = str(logical_name or "").casefold() - return ( - _ref_suffix(logical_name) == _DATAVERSE_REF_SUFFIX - or normalized == _DATAVERSE_RUNTIME_REF_LOGICAL_NAME.casefold() - ) - - def _host_of(url: str) -> str: """Return the host portion of an ``https://host/…`` URL for display.""" match = re.match(r"https?://([^/]+)", str(url).strip()) @@ -231,26 +215,27 @@ def _resolve_owner(props: dict) -> str: def _query_connection_references(runner): - """Return all Dataverse ``connectionreferences`` rows, or ``None`` when the - Dataverse token/endpoint is not available. - - Documented-tier read (Dataverse Web API v9.2) — no cassette required; tests - stub ``query_all``. + """Return the agent's connection references from the Declarative Agent + minimalBots components API, normalized to the row shape + ``_check_dv_connection`` consumes, or ``None`` when the AgentBuilder client + or the active-agent ``botId`` is unavailable. + + Validated-tier read (minimalBots ``POST …/components``). The same endpoint + and ``connectionReferenceChanges`` shape already back the shipped native + ``DA-CONN-001`` check (``checks/native_agent.py``); see + ``tests/fixtures/cassettes/INDEX.md`` and ``tests/mocks/ + agentbuilder_connectivity.py``. Fails loudly (lets the dispatcher degrade + this checkpoint to a WARNING) rather than overclaiming: an + ``AgentBuilderHTTPError`` propagates, and a 200 payload whose + ``connectionReferenceChanges`` is present but not a list raises + ``ValueError`` (mirrors ``native_agent._connection_references``). A missing + changeset is treated as "no references" (genuine absence), not an error. + + The fetch + normalize + fail-loudly logic is shared with ``ENV-004`` via + ``_da_connection_refs`` so the two DA connection-reference checks cannot + drift apart; this wrapper pins DV-CONN-001 to the single active agent. """ - env_url = getattr(runner, "env_url", None) - dv_token = getattr(runner, "dv_token", None) - if not env_url or not dv_token: - return None - # Belt-and-suspenders: keep scripts/ importable even if the module was - # imported before cli.py put it on the path. - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - return query_all( - env_url, - dv_token, - "connectionreferences", - "connectionreferenceid,connectionreferencelogicalname," - "connectionreferencedisplayname,connectorid,connectionid,statuscode", - ) + return read_active_agent_connection_references(runner) def _get_connections(runner): @@ -405,7 +390,7 @@ def _check_connection_auth(runner) -> list[CheckResult]: # ───────────────────────────────────────────────────────────────────── -# DV-CONN-001 — Dataverse connection reference binding (S5.4, PASS/FAIL). +# DV-CONN-001 — Workday SOAP connection reference binding (S5.4, PASS/FAIL). # ───────────────────────────────────────────────────────────────────── @@ -417,67 +402,44 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.SKIPPED.value, description=_DV_CONN_DESC, result=( - "Dataverse token not available — skipping the Dataverse " - "connection-reference check." + "AgentBuilder client or active-agent botId not available — " + "skipping the Workday connection-reference check." ), )] - dv_refs = [ - r - for r in refs - if str(r.get("connectorid") or "").lower().endswith( - _DATAVERSE_CONNECTOR_SUFFIX - ) - and _is_dataverse_runtime_ref( - r.get("connectionreferencelogicalname") - ) - ] - if len(dv_refs) > 1: - names = ", ".join( - sorted( - str(ref.get("connectionreferencelogicalname") or "(unnamed)") - for ref in dv_refs - ) - ) - return [CheckResult(roles=_MAKER_ROLES, - checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.WARNING.value, - description=_DV_CONN_DESC, - result=( - "Multiple ESS Dataverse connection references match the " - f"runtime and legacy package fingerprints: {names}. " - "FlightCheck cannot determine which reference is active." - ), - remediation=( - "Remove obsolete Workday package references, then rerun " - "FlightCheck against the remaining Dataverse binding." - ), - doc_link=_DOC_SIMPLIFIED, - )] - dv_ref = dv_refs[0] if dv_refs else None + wd_ref = next( + ( + r + for r in refs + if str(r.get("connectorid") or "") + .lower() + .endswith(_WORKDAY_CONNECTOR_SUFFIX) + ), + None, + ) - if dv_ref is None: + if wd_ref is None: return [CheckResult(roles=_MAKER_ROLES, checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.NOT_CONFIGURED.value, + priority=Priority.HIGH.value, status=Status.FAILED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"(\u2026_{_DATAVERSE_REF_SUFFIX}, connector " - "shared_commondataserviceforapps) was not found in this " - "environment." + "The ESS Workday SOAP connection reference (connector " + "shared_workdaysoap) was not found in the Declarative Agent " + "components payload." ), remediation=( - "Install/repair the Workday extension pack so its Dataverse " - "connection reference is created, then bind it to a Dataverse " - "connection you own." + "Install or repair the Workday extension pack so its Workday " + "SOAP connection reference is created, then bind it to a " + "Workday connection you own." ), doc_link=_DOC_SIMPLIFIED, )] - dv_ref_name = str(dv_ref.get("connectionreferencelogicalname")) - connection_id = dv_ref.get("connectionid") - statuscode = dv_ref.get("statuscode") + wd_ref_name = str( + wd_ref.get("connectionreferencelogicalname") or "(unnamed)" + ) + connection_id = wd_ref.get("connectionid") if not connection_id: return [CheckResult(roles=_MAKER_ROLES, @@ -485,31 +447,13 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.FAILED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is unbound " - "(connectionid=null)." - ), - remediation=( - "In Power Platform / Copilot Studio, bind the Dataverse " - "connection reference to an active Dataverse connection owned " - "by your own account." - ), - doc_link=_DOC_SIMPLIFIED, - )] - - if statuscode != 1: - return [CheckResult(roles=_MAKER_ROLES, - checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.FAILED.value, - description=_DV_CONN_DESC, - result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is bound but inactive " - f"(statuscode={statuscode})." + "The ESS Workday SOAP connection reference " + f"({wd_ref_name}) is unbound (connectionId=null)." ), remediation=( - "Re-authenticate or re-bind the Dataverse connection so its " - "status is active, using an account you own." + "In Power Platform / Copilot Studio, bind the Workday SOAP " + "connection reference to an active Workday connection owned by " + "your own account." ), doc_link=_DOC_SIMPLIFIED, )] @@ -529,9 +473,8 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.PASSED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is bound to an active " - "connection." + owner_note + "The ESS Workday SOAP connection reference " + f"({wd_ref_name}) is bound to a connection." + owner_note ), doc_link=_DOC_SIMPLIFIED, )] diff --git a/solutions/ess-maker-skills/scripts/flightcheck/registry.py b/solutions/ess-maker-skills/scripts/flightcheck/registry.py index dc759a9a1..3b852ac05 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/registry.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/registry.py @@ -31,9 +31,12 @@ **Scope:** the ESS + Workday *setup* checkpoints only — not the entire FlightCheck surface. Other integrations (ServiceNow ``SN-*``, graph -connector ``EXT-*``, ``SAP-*``) and the pre-existing ``ENV-003`` / -``ENV-004`` (+ detail) rows stay validated by the existing ``--scope`` -runs and are deliberately out of registry scope. See +connector ``EXT-*``, ``SAP-*``) and the pre-existing ``ENV-003`` row stay +validated by the existing ``--scope`` runs and are deliberately out of +registry scope. ``ENV-004`` was re-pointed to the Declarative Agent +minimalBots components + ALM API and is now registered here (clients +``AGENTBUILDER``) so it can run via the plan/``--checkpoint`` path like the +other DA checks. See ``plans/workday-setup/flightcheck-single-checkpoint.md``. """ @@ -540,18 +543,37 @@ class ResolvedPlan: priority=Priority.HIGH.value, roles=(Role.ESS_MAKER.value,), ), - # DV-CONN-001 — self-contained Dataverse read (its own connectionreferences - # query) plus a best-effort BAP owner echo. + # DV-CONN-001 — reads the Workday SOAP connection reference from the + # Declarative Agent minimalBots components API (AGENTBUILDER), plus a + # best-effort BAP owner echo (PP_ADMIN). CheckpointSpec( key="DV-CONN-001", category_fn=run_workday_extension_checks, category_label="Workday Extension", - clients=frozenset({DATAVERSE, PP_ADMIN}), + clients=frozenset({AGENTBUILDER, PP_ADMIN}), requires_config=True, - requires_dataverse_endpoint=True, + requires_dataverse_endpoint=False, priority=Priority.HIGH.value, roles=(Role.ESS_MAKER.value,), ), + # ENV-004 — re-pointed from the Dataverse connectionreference table to the + # Declarative Agent minimalBots components + ALM configure API + # (AGENTBUILDER). Reads every configured agent's connection references + # (bound/unbound) and, when an expected GRS commit SHA is configured, pins + # the deployed ALM commit (ENV-004-GRS). category_fn is + # run_environment_checks; the plan path filters emitted rows to the ENV-004 + # target (the ENV-004-UR-* / ENV-004-GRS detail rows are supplementary, and + # the GRS verdict also folds into the ENV-004 summary status). + CheckpointSpec( + key="ENV-004", + category_fn=run_environment_checks, + category_label="Environment", + clients=frozenset({AGENTBUILDER}), + requires_config=True, + requires_dataverse_endpoint=False, + priority=Priority.HIGH.value, + roles=(Role.POWER_PLATFORM_ADMIN.value, Role.ESS_MAKER.value), + ), # WD-REST-001 — pure config check (restBaseUrl trimmed to /api), no client. CheckpointSpec( key="WD-REST-001", diff --git a/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md b/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md index a973217b9..18cf38a3a 100644 --- a/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md +++ b/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md @@ -62,11 +62,9 @@ Workday → Entra Admin + Workday Admin). | ENV-001 | Power Platform environment exists | Critical | BAP Admin API | [prepare#set-up-your-power-platform-environment](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/prepare#set-up-your-power-platform-environment) | | ENV-002 | Dataverse database provisioned | Critical | BAP Admin API | [prepare#set-up-your-power-platform-environment](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/prepare#set-up-your-power-platform-environment) | | ENV-003 | Environment type | High | BAP Admin API | [prepare#set-up-your-power-platform-environment](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/prepare#set-up-your-power-platform-environment) | -| ENV-004 | Connections & connection references — binding + orphan detection, scoped to the references the ESS agent's enabled topics actually use (resolved via topic InvokeFlowAction flowIds → cloud flow `connectionReferences`). Skips when that scope can't be resolved; warns on a flow-listing API error. | High | BAP Admin API + Dataverse REST | [prepare#set-up-your-power-platform-environment](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/prepare#set-up-your-power-platform-environment) | -| ENV-004-OR-nnn | Orphan reference (in-scope reference points to a missing connection) | High | — | — | -| ENV-004-UR-nnn | Unbound reference (in-scope reference has no connection bound) | High | — | — | -| ENV-004-MR-nnn | Missing reference (agent flow uses a connection reference that does not exist in the environment) | High | — | — | -| ENV-004-UC-nnn | Unbound connection (no in-scope reference uses it; limited to the agent's connectors) | Medium | — | — | +| ENV-004 | Connections & connection references — reads `connectionReferenceChanges` from the Declarative Agent minimalBots components payload via `runner.agentbuilder.fetch_components`; unions and de-dupes references across all configured agents; reports whether each agent-declared connection reference is bound to a real connection. Skips without an AgentBuilder client or configured agent `botId`; warns on a components read error. | High | AgentBuilder (minimalBots components + ALM configure) | [prepare#set-up-your-power-platform-environment](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/prepare#set-up-your-power-platform-environment) | +| ENV-004-UR-nnn | Unbound reference (agent-declared reference has no connection bound) | High | — | — | +| ENV-004-GRS | Opt-in GRS commit pin. Skips unless `expectedGrsCommitSha` is configured in `.local/config.json`; reads `.../alm/{id}/configure`; passes when `commitSha` matches and fails on mismatch or missing `commitSha`. | High | AgentBuilder (minimalBots ALM configure) | — | | ENV-CAPACITY-001 | Copilot Studio message capacity provisioned | Critical | Power Platform Licensing API | [requirements-messages-management#prepaid-capacity](https://learn.microsoft.com/en-us/microsoft-copilot-studio/requirements-messages-management?tabs=new#prepaid-capacity) | | ENV-008 | DLP policies configured | High | BAP Admin API | [prepare#allow-the-external-systems-connector](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/prepare#allow-the-external-systems-connector) | diff --git a/tests/fixtures/cassettes/INDEX.md b/tests/fixtures/cassettes/INDEX.md index 9a52dd987..4ba88ec6b 100644 --- a/tests/fixtures/cassettes/INDEX.md +++ b/tests/fixtures/cassettes/INDEX.md @@ -87,7 +87,7 @@ table, it is not confirmed — see `tests/AGENTS.md` for what to do next. | Power Platform Admin (BAP) | `GET /providers/Microsoft.BusinessAppPlatform/scopes/admin/apiPolicies` | 200 | `flightcheck_pp_admin.yaml` | | PowerApps | `GET /providers/Microsoft.PowerApps/scopes/admin/environments/{env_id}/connections` | 200 | `flightcheck_pp_admin.yaml` | | Power Automate | `GET https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/scopes/admin/environments/{env_id}/v2/flows` (admin flow listing — lightweight summary: `apiId`, `state`, `workflowEntityId`, `workflowUniqueId`, `isManaged`. **Does NOT include `properties.connectionReferences`** — that block exists only in the per-flow detail below) | 200 | `flightcheck_flow_licensing.yaml` | -| Power Automate | `GET https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/scopes/admin/environments/{env_id}/flows/{flow_id}` (per-flow detail — `properties.connectionReferences..{connectionReferenceLogicalName, apiDefinition.properties.{tier,isCustomApi}}`. The `tier`/`isCustomApi` premium-custom signal is read inline by LIC-FLOW-001; the `connectionReferenceLogicalName` + connector are read by **ENV-004 agent-scoping** (`checks/_agent_connection_refs.py` calls `pp.get_flow` per topic-discovered flowId — same pattern as LIC-FLOW-001 — because the listing omits this block) to define which references ENV-004 judges) | 200 | `flightcheck_flow_licensing.yaml` | +| Power Automate | `GET https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/scopes/admin/environments/{env_id}/flows/{flow_id}` (per-flow detail — `properties.connectionReferences..{connectionReferenceLogicalName, apiDefinition.properties.{tier,isCustomApi}}`. The `tier`/`isCustomApi` premium-custom signal is read inline by LIC-FLOW-001. (ENV-004 no longer reads this endpoint: it was re-pointed to the Declarative Agent minimalBots components API — same surface as `DV-CONN-001` — and the former `checks/_agent_connection_refs.py` scoper was retired.) | 200 | `flightcheck_flow_licensing.yaml` | | Power Automate (runtime runs) | `GET https://api.flow.microsoft.com/providers/Microsoft.ProcessSimple/environments/{env_id}/flows/{flow_id}/runs` (run history — `properties.status` + `properties.response.name`; runtime/maker scope, NOT `/scopes/admin`) | 200 | `flightcheck_workday_runs.yaml` | | Dataverse (flow lifecycle) | `POST {env}/api/data/v9.2/workflows` (create cloud flow — category 5, `Prefer: return=representation` → `workflowid` in body + `OData-EntityId` header) | 201 | `flightcheck_infra003_flow.yaml` | | Dataverse (flow lifecycle) | `PATCH {env}/api/data/v9.2/workflows({id})` (activate — `{"statecode":1}`, `If-Match: *`) | 204 | `flightcheck_infra003_flow.yaml` | @@ -332,9 +332,9 @@ one auth flow, one cassette, simpler request shape. | Microsoft Graph v1.0 | `GET /v1.0/applications/{id}` | Entra: app exposes user_impersonation scope; preauthorized clients includes `4e4707ca-5f53-46a6-a819-f7765446e6ff` (Workday connector) | **No cassette required** — `validatable` via Graph CSDL. | | Microsoft Graph v1.0 | `GET /v1.0/oauth2PermissionGrants?$filter=clientId eq '...'` | Entra: admin consent granted for openid/profile/User.Read | **No cassette required** — `validatable` via Graph CSDL. | | Microsoft Graph v1.0 | `GET /v1.0/subscribedSkus` | License / SKU validation | **No cassette required** — `validatable` via Graph CSDL. | -| Dataverse | `GET /api/data/v9.2/connectionreferences?$select=connectionreferenceid,connectionreferencelogicalname,connectionreferencedisplayname,connectorid,connectionid,statuscode` | Workday integration **flavor detection**: deterministically distinguishes the OOTB simplified-Workday install (1 Workday connection reference shipped) from the full / legacy SOAP+custom install (3 Workday connection references shipped). Backs `WD-PKG-001` (package detection) and `WD-CONN-012` (package-aware connection-reference binding completeness). Same endpoint already powers `ENV-004` (general binding-state check in `flightcheck/checks/environment.py`). **ENV-004 now scopes its verdict** to the references the ESS agent actually uses — the env-wide list this endpoint returns is filtered to the logical names resolved by `checks/_agent_connection_refs.py` (topics → flowIds → cloud-flow `connectionReferences`), so placeholder/unbound references belonging to other apps or shipped-but-unused by a simplified install no longer FAIL the check. | **API contract remains `documented`** (response shape per the MS Learn `connectionreference` reference) — the cassette is not the contract evidence. **The cassettes ARE the fingerprint evidence**: the actual `connectionreferencelogicalname` suffixes shipped by Microsoft inside the simplified-install solution vs. the full/legacy solution are not on MS Learn. Both flavors use the same connector (`shared_workdaysoap`); the set of logical-name suffixes is the fingerprint. Captured by `tests/captures/record_dataverse_workday_connection_refs.py` against both a real OOTB-simplified tenant and a real full/legacy SOAP+custom tenant; redacted cassettes are committed at `dataverse_workday_connection_refs_simplified.yaml` (1 Workday ref: `_ff0df` / OAuthUser-OBO) and `dataverse_workday_connection_refs_full.yaml` (3 Workday refs: `_ff0df` / OAuthUser-OBO, `_0786a` / Generic User ISU, `_d6081` / Context Generic User ISU). `WD-PKG-001` matches against the trailing 5-hex suffix so the check is resilient to publisher-prefix changes. | +| Dataverse | `GET /api/data/v9.2/connectionreferences?$select=connectionreferenceid,connectionreferencelogicalname,connectionreferencedisplayname,connectorid,connectionid,statuscode` | Workday integration **flavor detection**: deterministically distinguishes the OOTB simplified-Workday install (1 Workday connection reference shipped) from the full / legacy SOAP+custom install (3 Workday connection references shipped). Backs `WD-PKG-001` (package detection) and `WD-CONN-012` (package-aware connection-reference binding completeness). This endpoint no longer powers `ENV-004`: ENV-004 was re-pointed to the Declarative Agent minimalBots components API (`connectionReferenceChanges`, the same surface as `DV-CONN-001`), reading each configured agent's declared references directly, and the former Dataverse-scoping via `checks/_agent_connection_refs.py` was retired. | **API contract remains `documented`** (response shape per the MS Learn `connectionreference` reference) — the cassette is not the contract evidence. **The cassettes ARE the fingerprint evidence**: the actual `connectionreferencelogicalname` suffixes shipped by Microsoft inside the simplified-install solution vs. the full/legacy solution are not on MS Learn. Both flavors use the same connector (`shared_workdaysoap`); the set of logical-name suffixes is the fingerprint. Captured by `tests/captures/record_dataverse_workday_connection_refs.py` against both a real OOTB-simplified tenant and a real full/legacy SOAP+custom tenant; redacted cassettes are committed at `dataverse_workday_connection_refs_simplified.yaml` (1 Workday ref: `_ff0df` / OAuthUser-OBO) and `dataverse_workday_connection_refs_full.yaml` (3 Workday refs: `_ff0df` / OAuthUser-OBO, `_0786a` / Generic User ISU, `_d6081` / Context Generic User ISU). `WD-PKG-001` matches against the trailing 5-hex suffix so the check is resilient to publisher-prefix changes. | | Dataverse | `GET /api/data/v9.2/botcomponents?$filter=name eq 'Workday [System] - 1: Set User Context V2'` | Required Workday topic installed | **No cassette required** — `documented`; verify against MS Learn `botcomponent` reference. | -| Dataverse | `GET /api/data/v9.2/botcomponents?$select=name,schemaname,data&$filter=componenttype eq 9` | `WD-REF-001` reference-data availability: reads all topics to reconcile the reference picklists each topic REQUESTS from GetReferenceData (`referenceDataKey: KEY`) against the keys GetReferenceData SUPPORTS (`referenceDataKey = "KEY"` switch). Also consumed by **ENV-004 agent-scoping** (`checks/_agent_connection_refs.py`), which adds `_parentbotid_value eq '{botId}' and statecode eq 0` to the same endpoint to read only the ESS agent's **enabled** topics and regex-extract their InvokeFlowAction `flowId:` GUIDs — the seed for resolving which connection references the agent actually uses. | **No cassette required** — `documented`; `botcomponents.data` is the topic YAML per the MS Learn `botcomponent` reference. Tests stub `query_all`. | +| Dataverse | `GET /api/data/v9.2/botcomponents?$select=name,schemaname,data&$filter=componenttype eq 9` | `WD-REF-001` reference-data availability: reads all topics to reconcile the reference picklists each topic REQUESTS from GetReferenceData (`referenceDataKey: KEY`) against the keys GetReferenceData SUPPORTS (`referenceDataKey = "KEY"` switch). (No longer consumed by ENV-004: that check was re-pointed to the Declarative Agent minimalBots components API and the `checks/_agent_connection_refs.py` scoper was retired.) | **No cassette required** — `documented`; `botcomponents.data` is the topic YAML per the MS Learn `botcomponent` reference. Tests stub `query_all`. | --- diff --git a/tests/flightcheck/checks/test_agent_connection_refs.py b/tests/flightcheck/checks/test_agent_connection_refs.py deleted file mode 100644 index 1bdfc7bbb..000000000 --- a/tests/flightcheck/checks/test_agent_connection_refs.py +++ /dev/null @@ -1,330 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -"""Unit tests for the ENV-004 agent connection-reference scope builder -(``flightcheck.checks._agent_connection_refs.build_agent_ref_scope``). - -The builder resolves the connection references an ESS agent actually -uses so ENV-004 can stop judging environment-wide references (which -produced false FAILs on other apps' refs and on the ESS-shipped -placeholder refs that ship unbound-by-design on a Workday simplified -install). - -Chain under test: config botId(s) -> Dataverse ``botcomponents`` -(enabled topics) ``data`` -> InvokeFlowAction flowIds -> per-flow -``pp.get_flow`` detail -> that detail's ``connectionReferences``. - -The Dataverse ``botcomponents`` read is ``documented`` tier (tests stub -``query_all``); the BAP per-flow detail is ``validated`` tier — its -record shape (``properties.connectionReferences..{connectionReferenceLogicalName,apiDefinition}``) -comes from ``tests/mocks/pp_admin.py`` (``MOCK_STATUS = "validated"``, -cassette ``flightcheck_flow_licensing.yaml``). The listing -(``pp.get_flows``) is deliberately NOT used: it omits -``connectionReferences`` entirely, which would yield an empty scope and -silently turn ENV-004 into a no-op. -""" - -from __future__ import annotations - -from types import SimpleNamespace -from typing import Any - -import pytest - -from tests.conftest import require_validated_mock -from tests.mocks import pp_admin as pp - -require_validated_mock(pp) - - -FLOW_A = "11111111-1111-1111-1111-111111111111" -FLOW_B = "22222222-2222-2222-2222-222222222222" - - -def _topic(data: str) -> dict[str, Any]: - """A ``botcomponents`` topic row as ENV-004 scoping reads it.""" - return {"name": "SomeTopic", "schemaname": "cr123_sometopic", "data": data} - - -def _invoke(flow_id: str) -> str: - """Topic YAML fragment invoking a cloud flow (Pattern B).""" - return ( - "kind: AdaptiveDialog\n" - "actions:\n" - " - kind: InvokeFlowAction\n" - f" flowId: {flow_id}\n" - ) - - -def _detail_with_ref(flow_id: str, api_name: str) -> dict[str, Any]: - """A BAP per-flow DETAIL record binding a flow to one connector's - ref, built from the validated ``pp_admin`` mock builders. The - connection references live in the DETAIL response (``get_flow``), not - the listing.""" - return pp.flow_detail( - flow_id=flow_id, - connection_refs={api_name: pp.flow_connector_ref(api_name=api_name)}, - ) - - -class _FakePP: - """Fake ``pp_admin`` exposing ``get_flow(env_id, flow_id)`` -> detail. - - ``details`` maps a flow_id to its DETAIL dict (or an ``{"_error", - "_status"}`` payload). A flow_id absent from the map returns ``None`` - — modelling a flow not visible on the admin surface, exactly as - ``pp_admin.get_flow`` does for a 404. - """ - - def __init__(self, details=None): - self._details = details or {} - self.calls: list[tuple[str, str]] = [] - - def get_flow(self, env_id, flow_id): - self.calls.append((env_id, flow_id)) - return self._details.get(flow_id) - - def get_flows(self, _env_id): # pragma: no cover - guard only - raise AssertionError( - "build_agent_ref_scope must read connection references from the " - "per-flow DETAIL (get_flow), not the listing (get_flows) — the " - "listing omits connectionReferences (flightcheck_flow_licensing.yaml)." - ) - - -def _runner(*, config, details=None): - return SimpleNamespace( - config=config, - env_url="https://example.crm.dynamics.com", - dv_token="dv-token", - env_id="env-1", - pp_admin=_FakePP(details or {}), - ) - - -def test_scope_resolves_logical_names_and_connectors(monkeypatch): - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = _runner( - config={"agents": [{"botId": "bot-1"}]}, - details={FLOW_A: _detail_with_ref(FLOW_A, "shared_commondataserviceforapps")}, - ) - - scope = mod.build_agent_ref_scope(runner) - - assert scope is not None - assert scope.logical_names == frozenset({"ref_shared_commondataserviceforapps"}) - assert scope.connectors == frozenset({"shared_commondataserviceforapps"}) - - -def test_scope_reads_from_flow_detail_not_listing(monkeypatch): - """Regression guard: the builder resolves refs via the per-flow - DETAIL (``get_flow``), never the listing. ``_FakePP.get_flows`` - raises, so any lapse back to the listing fails loudly here.""" - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = _runner( - config={"agents": [{"botId": "bot-1"}]}, - details={FLOW_A: _detail_with_ref(FLOW_A, "shared_workdaysoap")}, - ) - - scope = mod.build_agent_ref_scope(runner) - - assert scope is not None - # get_flow was called with the topic-discovered flowId directly. - assert runner.pp_admin.calls == [("env-1", FLOW_A)] - - -def test_scope_none_without_botid(monkeypatch): - """No configured agent botId -> cannot scope -> None (caller SKIPs).""" - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = _runner( - config={}, details={FLOW_A: _detail_with_ref(FLOW_A, "shared_workdaysoap")} - ) - - assert mod.build_agent_ref_scope(runner) is None - - -def test_scope_supports_single_agent_config(monkeypatch): - """The legacy single-agent ``config['agent']`` shape is honored.""" - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = _runner( - config={"agent": {"botId": "solo"}}, - details={FLOW_A: _detail_with_ref(FLOW_A, "shared_workdaysoap")}, - ) - - scope = mod.build_agent_ref_scope(runner) - - assert scope is not None - assert "ref_shared_workdaysoap" in scope.logical_names - - -def test_scope_none_when_topics_invoke_no_flows(monkeypatch): - """Topics with no InvokeFlowAction flowIds -> None (can't scope).""" - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr( - mod, "query_all", lambda *a, **k: [_topic("kind: SendActivity\n")] - ) - runner = _runner( - config={"agents": [{"botId": "bot-1"}]}, - details={FLOW_A: _detail_with_ref(FLOW_A, "shared_workdaysoap")}, - ) - - assert mod.build_agent_ref_scope(runner) is None - - -def test_scope_none_when_flow_detail_not_found(monkeypatch): - """flowIds discovered but the admin surface returns none of them - (get_flow -> None) -> scoping unreliable -> None rather than - under-report.""" - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = _runner( - config={"agents": [{"botId": "bot-1"}]}, - details={}, # FLOW_A not present -> get_flow returns None - ) - - assert mod.build_agent_ref_scope(runner) is None - - -def test_scope_raises_on_flow_detail_auth_error(monkeypatch): - """A 401/403 ``_error`` payload from a flow detail fetch surfaces - loudly (caller converts to WARNING) — not a silent SKIP.""" - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = _runner( - config={"agents": [{"botId": "bot-1"}]}, - details={FLOW_A: {"_error": "403 Forbidden", "_status": 403}}, - ) - - with pytest.raises(RuntimeError, match="unauthorized"): - mod.build_agent_ref_scope(runner) - - -def test_scope_skips_unreadable_flow_but_keeps_readable_ones(monkeypatch): - """A non-auth unreadable flow (404 / other _error) is skipped, while - a sibling readable flow still contributes its refs.""" - from flightcheck.checks import _agent_connection_refs as mod - - def _fake(env_url, token, entity_set, select, filter_expr=None): - return [_topic(_invoke(FLOW_A) + _invoke(FLOW_B))] - - monkeypatch.setattr(mod, "query_all", _fake) - runner = _runner( - config={"agents": [{"botId": "bot-1"}]}, - details={ - FLOW_A: _detail_with_ref(FLOW_A, "shared_workdaysoap"), - FLOW_B: {"_error": "not found", "_status": 404}, - }, - ) - - scope = mod.build_agent_ref_scope(runner) - - assert scope is not None - assert scope.logical_names == frozenset({"ref_shared_workdaysoap"}) - - -def test_scope_raises_on_flow_detail_exception(monkeypatch): - """A raised exception during a flow detail fetch surfaces loudly.""" - from flightcheck.checks import _agent_connection_refs as mod - - class _BoomPP: - def get_flow(self, env_id, flow_id): - raise ConnectionError("socket reset") - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = SimpleNamespace( - config={"agents": [{"botId": "bot-1"}]}, - env_url="https://example.crm.dynamics.com", - dv_token="dv-token", - env_id="env-1", - pp_admin=_BoomPP(), - ) - - with pytest.raises(RuntimeError, match="socket reset"): - mod.build_agent_ref_scope(runner) - - -def test_scope_queries_enabled_topics_only(monkeypatch): - """The topic query MUST filter to enabled topics (statecode 0) scoped - to the configured botId — a ref only reachable through a disabled - topic is not a live runtime dependency.""" - from flightcheck.checks import _agent_connection_refs as mod - - captured: dict[str, Any] = {} - - def _fake(env_url, token, entity_set, select, filter_expr=None): - captured["entity_set"] = entity_set - captured["filter"] = filter_expr - return [_topic(_invoke(FLOW_A))] - - monkeypatch.setattr(mod, "query_all", _fake) - runner = _runner( - config={"agents": [{"botId": "bot-xyz"}]}, - details={FLOW_A: _detail_with_ref(FLOW_A, "shared_workdaysoap")}, - ) - - mod.build_agent_ref_scope(runner) - - assert captured["entity_set"] == "botcomponents" - assert "statecode eq 0" in captured["filter"] - assert "componenttype eq 9" in captured["filter"] - assert "bot-xyz" in captured["filter"] - - -def test_scope_unions_across_multiple_agents(monkeypatch): - """Refs are unioned across every configured agent botId.""" - from flightcheck.checks import _agent_connection_refs as mod - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if "bot-1" in (filter_expr or ""): - return [_topic(_invoke(FLOW_A))] - if "bot-2" in (filter_expr or ""): - return [_topic(_invoke(FLOW_B))] - return [] - - monkeypatch.setattr(mod, "query_all", _fake) - runner = _runner( - config={"agents": [{"botId": "bot-1"}, {"botId": "bot-2"}]}, - details={ - FLOW_A: _detail_with_ref(FLOW_A, "shared_commondataserviceforapps"), - FLOW_B: _detail_with_ref(FLOW_B, "shared_workdaysoap"), - }, - ) - - scope = mod.build_agent_ref_scope(runner) - - assert scope is not None - assert scope.logical_names == frozenset( - {"ref_shared_commondataserviceforapps", "ref_shared_workdaysoap"} - ) - assert scope.connectors == frozenset( - {"shared_commondataserviceforapps", "shared_workdaysoap"} - ) - - -def test_scope_matched_flow_without_refs_is_empty_not_none(monkeypatch): - """A readable flow that carries no connection references yields an - empty (but resolved) scope — distinct from the unresolvable None.""" - from flightcheck.checks import _agent_connection_refs as mod - - monkeypatch.setattr(mod, "query_all", lambda *a, **k: [_topic(_invoke(FLOW_A))]) - runner = _runner( - config={"agents": [{"botId": "bot-1"}]}, - details={FLOW_A: pp.flow_detail(flow_id=FLOW_A, connection_refs={})}, - ) - - scope = mod.build_agent_ref_scope(runner) - - assert scope is not None - assert scope.logical_names == frozenset() - assert scope.connectors == frozenset() diff --git a/tests/flightcheck/checks/test_env_004_remediation_links.py b/tests/flightcheck/checks/test_env_004_remediation_links.py index 59ec39320..4beae673f 100644 --- a/tests/flightcheck/checks/test_env_004_remediation_links.py +++ b/tests/flightcheck/checks/test_env_004_remediation_links.py @@ -1,20 +1,24 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Tests for ENV-004 connection-binding remediations. - -The user surfaced that ENV-004 remediations had no actionable link — -operators were told to "fix the broken connection reference" with no -pointer to the actual page in the maker portal. This module pins the -deep links that the remediation strings must now carry, for every -ENV-004 row that asks the operator to take a manual action. - -Strategy: -- Drive `_check_connections_and_refs` with a tiny in-memory fake of - the PPAdminClient + a monkeypatched `auth.query_all`. -- Build the three buggy bindings the check is supposed to surface - (orphan ref, unbound ref, unbound connection) and assert each - remediation contains the expected env-scoped maker URL. +"""Tests for ENV-004 (Declarative Agent connection references + GRS commit pin). + +ENV-004 was re-pointed from the Dataverse ``connectionreference`` table to the +Declarative Agent minimalBots components + ALM configure API (validated tier; +the same ``connectionReferenceChanges`` shape the shipped native ``DA-CONN-001`` +check consumes). This module drives ``_check_connections_and_refs`` directly +against a faked ``runner.agentbuilder`` built from the validated +``agentbuilder_connectivity`` mocks. + +Two halves are covered: + * Connection-reference classification — bound (PASS) vs unbound (FAIL), the + per-ref ``ENV-004-UR-*`` detail rows, and the SKIP/WARNING guards. + * GRS commit pin (``ENV-004-GRS``) — opt-in SKIP, PASS on a matching commit, + FAIL on mismatch, WARNING on a configure read error, and the fold of the + GRS verdict into the ENV-004 summary status. + +Per tests/AGENTS.md, every GOOD/BAD/WARNING assertion pins a phrase from both +``result`` and ``remediation``. """ from __future__ import annotations @@ -25,10 +29,15 @@ import pytest +from tests.conftest import require_validated_mock +from tests.mocks import agentbuilder_connectivity as ab + +require_validated_mock(ab) + @pytest.fixture(autouse=True) def _scripts_on_path(): - """Make `flightcheck.*` and `auth` importable from the kit's scripts dir.""" + """Make ``flightcheck.*`` importable from the kit's scripts dir.""" repo_root = Path(__file__).resolve().parents[2] scripts_dir = repo_root / "solutions" / "ess-maker-skills" / "scripts" sys.path.insert(0, str(scripts_dir)) @@ -41,1089 +50,345 @@ def _scripts_on_path(): pass -def test_maker_connections_url_targets_powerautomate(): - from flightcheck.checks._maker_urls import maker_connections_url - - assert maker_connections_url("env-123") == ( - "https://make.powerautomate.com/environments/env-123/connections" - ) - - -def test_maker_solutions_url_targets_powerapps(): - """Solutions are only surfaced in the Power Apps maker — the - Power Automate maker does not expose the Connection References - pane that ENV-004 asks the operator to open.""" - from flightcheck.checks._maker_urls import maker_solutions_url - - assert maker_solutions_url("env-123") == ( - "https://make.powerapps.com/environments/env-123/solutions" - ) - - -class _FakePPAdmin: - """Minimal stand-in for ``PPAdminClient`` covering the methods - ENV-004 calls. Returns the list passed at construction time.""" - - def __init__(self, connections): - self._connections = connections - - def get_connections(self, _env_id): - return self._connections - - -def _make_runner(connections): - return SimpleNamespace( - pp_admin=_FakePPAdmin(connections), - env_id="env-deeplinks", - env_url="https://example.crm.dynamics.com", - dv_token="fake-token", - ) +# ───────────────────────────────────────────────────────────────────── +# Fakes. The check reads only runner.agentbuilder (fetch_components + +# get_realm_configuration), runner.config (agent botId + GRS keys), and +# runner.env_id (best-effort remediation deep link). +# ───────────────────────────────────────────────────────────────────── +_MISSING = object() -def _conn(name, display_name="Display Name"): - """Shape we need from PP Admin's get_connections — just `name` - (the GUID id field) and `properties.displayName`.""" - return {"name": name, "properties": {"displayName": display_name, "apiId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap"}} - - -def _ref(conn_id, ref_id=None, display="Reference", solution_id=None): - """Shape we need from Dataverse connectionreferences query. - - ``solution_id`` does NOT live on the ref itself in the production - schema — connectionreference has no solution column. We carry it - on the fake dict purely so the test fixtures (``_patch_query_all``) - can derive a solutioncomponents response from the same input list, - keeping each test's setup compact. - """ - return { - "connectionreferenceid": ref_id or "ref-id", - "connectionreferencelogicalname": "logical_name", - "connectionreferencedisplayname": display, - "connectorid": "shared_workdaysoap", - "connectionid": conn_id, - "statuscode": 1, - # NOTE: prefixed so the production code path never reads this - # — it's test-fixture metadata, not a Dataverse column. - "_test_solution_id": solution_id or "00000000-0000-0000-0000-000000009000", - } - - -def _solution(sid, friendly_name, unique_name=None, ismanaged=False): - """Shape we need from Dataverse `solutions` query for the - ref → solution lookup. Defaults to ``ismanaged=False`` (unmanaged) - because that's the layer the maker can actually edit and the one - production prefers when picking among multiple matches.""" - return { - "solutionid": sid, - "uniquename": unique_name or friendly_name.lower().replace(" ", ""), - "friendlyname": friendly_name, - "ismanaged": ismanaged, - } - - -def _patch_query_all(monkeypatch, env_mod, *, conn_refs, solutions=None): - """Install a dispatching fake for ``auth.query_all`` covering all - three queries ENV-004 issues: - - 1. ``connectionreferences`` → returns the conn_refs argument - 2. ``solutioncomponents`` → derives a (ref_id → solution_id) - mapping from each ref's ``_test_solution_id`` fixture field - 3. ``solutions`` → returns the solutions argument - - Tests that don't care about the per-row deep link can omit - ``solutions`` and the third call returns []; the production code - falls back to the env-wide solutions URL. - """ - solutions = solutions or [] - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - # Production code never reads `_test_solution_id` — strip - # it so the mock can't accidentally make production logic - # depend on a non-existent Dataverse column. - return [{k: v for k, v in r.items() if not k.startswith("_test_")} - for r in conn_refs] - if entity_set == "solutioncomponents": - return [ - {"objectid": r["connectionreferenceid"], - "_solutionid_value": r["_test_solution_id"]} - for r in conn_refs if r.get("_test_solution_id") - ] - if entity_set == "solutions": - return solutions - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - -@pytest.fixture(autouse=True) -def _scope_to_all_present_refs(monkeypatch): - """Reproduce ENV-004's original whole-environment behavior for the - remediation-link tests in this module. - - ENV-004 now scopes its verdict to the connection references the agent - actually uses (``build_agent_ref_scope``). These tests predate that - scoping and assert on the binding-state / deep-link behavior, so we - default the scope to "every reference + connector present in the - test's fixtures", which keeps all of a test's refs in scope exactly - as the pre-scoping code judged them. Tests exercising the new - scoping / SKIP / missing-ref paths override ``build_agent_ref_scope`` - themselves (a later ``monkeypatch.setattr`` wins).""" - from flightcheck.checks import environment as env_mod - from flightcheck.checks._agent_connection_refs import AgentRefScope - from flightcheck.checks._dlp_utils import normalize_connector_id - - def _all_present(runner): - refs = env_mod.query_all( - runner.env_url, runner.dv_token, "connectionreferences", "sel" - ) - logicals = { - (r.get("connectionreferencelogicalname") or "").lower() - for r in refs or [] - if (r.get("connectionreferencelogicalname") or "") - } - connectors = { - normalize_connector_id(r.get("connectorid")) - for r in refs or [] - if r.get("connectorid") - } - try: - conns = runner.pp_admin.get_connections(runner.env_id) - except Exception: - conns = [] - for c in conns or []: - cid = normalize_connector_id((c.get("properties", {}) or {}).get("apiId")) - if cid: - connectors.add(cid) - return AgentRefScope( - logical_names=frozenset(logicals), - connectors=frozenset(connectors), +class _FakeAgentBuilder: + def __init__( + self, + *, + components=_MISSING, + configuration=None, + configure_error=None, + ): + self._components = ( + ab.components_with_references() + if components is _MISSING + else components ) + self._configuration = configuration + self._configure_error = configure_error - monkeypatch.setattr(env_mod, "build_agent_ref_scope", _all_present) + def fetch_components(self, _agent_id): + return self._components + def get_realm_configuration(self, _agent_id, realm): + # Mirror the real client's contract so a test can't pass an int-typed + # realm the production code would reject. + if type(realm) is not int: + raise ValueError("Realm must be the numeric Dev/Test/Prod value.") + if self._configure_error is not None: + raise self._configure_error + return self._configuration -def _scope(*, logical_names, connectors=("shared_workdaysoap",)): - """Build a fixed :class:`AgentRefScope` for the agent-scoping tests.""" - from flightcheck.checks._agent_connection_refs import AgentRefScope - - return AgentRefScope( - logical_names=frozenset(logical_names), - connectors=frozenset(connectors), - ) - -def _ref_named(logical, conn_id, *, ref_id, display="Reference"): - """A connectionreferences row with an explicit logical name (the - default ``_ref`` helper hardcodes 'logical_name').""" - return { - "connectionreferenceid": ref_id, - "connectionreferencelogicalname": logical, - "connectionreferencedisplayname": display, - "connectorid": "shared_workdaysoap", - "connectionid": conn_id, - "statuscode": 1, - } - - -def _conn_c(name, connector, display_name="Display Name"): - """A get_connections record with an explicit connector.""" - return { - "name": name, - "properties": { - "displayName": display_name, - "apiId": f"/providers/Microsoft.PowerApps/apis/{connector}", - }, - } - - -def test_env_004_skips_when_agent_scope_unresolvable(monkeypatch): - """When the agent's connection-reference set can't be resolved - (builder returns None), ENV-004 SKIPs rather than judging every - reference in the environment.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("", display="Placeholder")]) - monkeypatch.setattr(env_mod, "build_agent_ref_scope", lambda runner: None) - - results = env_mod._check_connections_and_refs(runner) - - env004 = [r for r in results if r.checkpoint_id == "ENV-004"] - assert len(env004) == 1 - assert env004[0].status == "Skipped" - assert "belong to this agent" in env004[0].result - # The unbound placeholder ref must NOT surface as a FAILED detail row. - assert not any(r.checkpoint_id.startswith("ENV-004-UR-") for r in results) - - -def test_env_004_warns_when_scope_build_raises(monkeypatch): - """A genuine API error while resolving the agent's flows surfaces as - a WARNING (principle 3), not a silent SKIP or an env-wide verdict.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("real-conn-id")]) - - def _boom(runner): - raise RuntimeError("flow listing failed: 403") - - monkeypatch.setattr(env_mod, "build_agent_ref_scope", _boom) - - results = env_mod._check_connections_and_refs(runner) - - env004 = [r for r in results if r.checkpoint_id == "ENV-004"] - assert len(env004) == 1 - assert env004[0].status == "Warning" - assert "403" in env004[0].result - - -def test_env_004_out_of_scope_unbound_ref_is_not_failed(monkeypatch): - """The customer-reported bug: an unbound reference the agent does NOT - use (e.g. the ESS-shipped placeholder on a simplified install) must - not FAIL ENV-004. Only the agent's own reference is judged.""" - from flightcheck.checks import environment as env_mod - - used = _ref_named("msdyn_used", "conn-1", ref_id="ref-used", display="Used") - placeholder = _ref_named("msdyn_placeholder", "", ref_id="ref-ph", display="Placeholder") - runner = _make_runner(connections=[_conn("conn-1")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [used, placeholder]) - monkeypatch.setattr( - env_mod, "build_agent_ref_scope", - lambda runner: _scope(logical_names={"msdyn_used"}), - ) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - assert summary.status == "Passed", summary.__dict__ - # The out-of-scope placeholder must not produce an unbound-ref FAIL. - assert not any(r.checkpoint_id.startswith("ENV-004-UR-") for r in results) - assert "1 reference(s) used by this agent" in summary.result - - -def test_env_004_missing_ref_emits_mr_row(monkeypatch): - """A logical name the agent's flows reference but that has no row in - the environment surfaces as an ENV-004-MR-* FAILED detail row.""" - from flightcheck.checks import environment as env_mod - - used = _ref_named("msdyn_used", "conn-1", ref_id="ref-used") - runner = _make_runner(connections=[_conn("conn-1")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [used]) - monkeypatch.setattr( - env_mod, "build_agent_ref_scope", - lambda runner: _scope(logical_names={"msdyn_used", "msdyn_missing"}), - ) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - assert summary.status == "Failed", summary.__dict__ - mr = [r for r in results if r.checkpoint_id.startswith("ENV-004-MR-")] - assert len(mr) == 1 - assert mr[0].status == "Failed" - assert "msdyn_missing" in mr[0].result - assert "no such reference exists" in mr[0].result - assert "1 missing ref(s)" in summary.result - - -def test_env_004_unbound_connection_scoped_to_agent_connectors(monkeypatch): - """Unbound connections are only flagged when their connector is one - the agent uses — a stale connection of an unrelated connector in the - same environment is not reported.""" - from flightcheck.checks import environment as env_mod - - bound = _conn_c("bound-wd", "shared_workdaysoap") - stale_wd = _conn_c("stale-wd", "shared_workdaysoap", display_name="Stale Workday") - other = _conn_c("other-app", "shared_office365", display_name="Other App") - used = _ref_named("msdyn_used", "bound-wd", ref_id="ref-used") - - runner = _make_runner(connections=[bound, stale_wd, other]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [used]) - monkeypatch.setattr( - env_mod, "build_agent_ref_scope", - lambda runner: _scope(logical_names={"msdyn_used"}, connectors={"shared_workdaysoap"}), +def _runner(*, agentbuilder=None, config=_MISSING, env_id="env-deeplinks"): + return SimpleNamespace( + agentbuilder=agentbuilder, + config=( + {"agent": {"botId": ab.MOCK_AGENT_ID}} + if config is _MISSING + else config + ), + env_id=env_id, ) - results = env_mod._check_connections_and_refs(runner) - - uc_rows = [r for r in results if r.checkpoint_id.startswith("ENV-004-UC-")] - # Only the same-connector stale Workday connection is flagged. - assert len(uc_rows) == 1 - assert "Stale Workday" in uc_rows[0].description - assert not any("Other App" in r.description for r in uc_rows) - - -def test_env_004_summary_orphan_remediation_links_to_solutions(monkeypatch): - """The top-level ENV-004 row, when orphan refs are present, must - point the operator at the Power Apps Solutions page where the - Connection References pane lives.""" - from flightcheck.checks import environment as env_mod - - # 1 connection that exists, 1 ref pointing at a DIFFERENT (missing) connection. - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("missing-conn-id", display="Workday")]) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - assert summary.status == "Failed", summary.__dict__ - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in (summary.remediation or "") - - -def test_env_004_summary_unbound_conns_links_to_connections_list(monkeypatch): - """When only unbound (extra) connections exist with no orphan refs, - the top-level row must link to the env-scoped connections list.""" - from flightcheck.checks import environment as env_mod - - # 2 connections, only the first has a ref bound to it -> second is unbound. - runner = _make_runner(connections=[_conn("bound-conn"), _conn("unbound-conn")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("bound-conn")]) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - assert summary.status == "Warning", summary.__dict__ - assert "https://make.powerautomate.com/environments/env-deeplinks/connections" in (summary.remediation or "") - - -def test_env_004_orphan_ref_detail_links_to_solutions(monkeypatch): - """Each ENV-004-OR-* (orphan ref) detail row must carry the - solutions deep link so the operator can re-bind the reference.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("missing-conn-id", display="Workday")]) - - results = env_mod._check_connections_and_refs(runner) - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - assert orphan.status == "Failed" - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in (orphan.remediation or "") +def _check(runner): + from flightcheck.checks.environment import _check_connections_and_refs - -def test_env_004_unbound_ref_detail_links_to_solutions(monkeypatch): - """ENV-004-UR-* (ref with no connectionid set) detail must point - the operator at Solutions to bind the reference.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[]) - # connectionid="" -> classified as unbound ref - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("", display="UnboundRef")]) - - results = env_mod._check_connections_and_refs(runner) - - unbound = next(r for r in results if r.checkpoint_id.startswith("ENV-004-UR-")) - assert unbound.status == "Failed" - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in (unbound.remediation or "") - - -def test_env_004_unbound_conn_detail_links_to_connections_list(monkeypatch): - """ENV-004-UC-* (connection no ref points at) detail must link to - the connections list so the operator can remove the stale entry.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("orphan-conn", display_name="Stale Connection")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: []) - - results = env_mod._check_connections_and_refs(runner) - - uc = next(r for r in results if r.checkpoint_id.startswith("ENV-004-UC-")) - assert uc.status == "Warning" - assert "https://make.powerautomate.com/environments/env-deeplinks/connections" in (uc.remediation or "") - - -# --------------------------------------------------------------------------- -# ENV-004-UC-* honesty: the check ONLY knows the connection isn't -# referenced by the agent's own solution. It can't see flows, canvas -# apps, or other solutions in the env. The original prose ("If unused, -# remove '' ... to reduce clutter") under-warned operators about -# the silent-breakage risk of deleting a connection used elsewhere AND -# gave no way to verify true disuse. The new prose must: -# 1) reframe the finding as scoped to THIS agent's solution -# 2) explicitly warn that deletion can silently break dependents -# 3) walk through concrete verification steps (Power Apps connection -# detail, Power Automate flows, other solutions' connection refs) -# 4) gate the "delete" action on the operator's verification -# --------------------------------------------------------------------------- - - -def test_env_004_unbound_conn_result_scopes_finding_to_this_solution(monkeypatch): - """Old result said 'No reference uses this connection' — which over- - claims, because the check only inspects the agent's own solution. - The new wording must scope to this agent's solution so the operator - doesn't read it as 'nothing in the env uses this'.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("orphan-conn", display_name="Stale Connection")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: []) - - uc = next( - r for r in env_mod._check_connections_and_refs(runner) - if r.checkpoint_id.startswith("ENV-004-UC-") - ) - assert "agent's solution" in uc.result.lower(), uc.result - # The original over-claim string must not regress. - assert "no reference uses this connection" not in uc.result.lower(), uc.result + return _check_connections_and_refs(runner) -def test_env_004_unbound_conn_remediation_warns_about_silent_breakage(monkeypatch): - """Deleting a connection that's used by another resource fails - silently from the operator's perspective (the dependent breaks on - next run, not at delete time). The remediation must call this out - so the operator doesn't treat 'delete to reduce clutter' as safe.""" - from flightcheck.checks import environment as env_mod +def _by_id(results): + return {r.checkpoint_id: r for r in results} - runner = _make_runner(connections=[_conn("orphan-conn", display_name="Stale Connection")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: []) - uc = next( - r for r in env_mod._check_connections_and_refs(runner) - if r.checkpoint_id.startswith("ENV-004-UC-") +def _bound(connector="shared_service-now", connection_id="conn-good"): + return ab.connection_reference_change( + connector=connector, connection_id=connection_id ) - rem = (uc.remediation or "").lower() - # Must explicitly say verification is required before deletion. - assert "verify" in rem, rem - # Must explicitly warn about silent breakage. - assert ("silently" in rem) or ("does not warn" in rem), rem - # The old phrasing was actively misleading and must not regress. - assert "to reduce clutter" not in rem, rem - - -def test_env_004_unbound_conn_remediation_lists_three_verification_paths(monkeypatch): - """Verification requires checking three places the check itself - didn't look. All three must be named — naming only one would imply - the others aren't relevant, and the operator would skip them.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("orphan-conn", display_name="Workday")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: []) - - uc = next( - r for r in env_mod._check_connections_and_refs(runner) - if r.checkpoint_id.startswith("ENV-004-UC-") - ) - rem = uc.remediation or "" - # 1. Power Automate Connections detail page (shows app dependencies). - assert "Connections" in rem and "make.powerautomate.com/environments/env-deeplinks/connections" in rem, rem - # 2. Power Automate flows (the only place flow dependencies show up). - assert "Power Automate" in rem, rem - assert "make.powerautomate.com/environments/env-deeplinks/flows" in rem, rem - # 3. Other solutions' connection references (the connection could be - # bound by a connection reference in a different solution). - assert "Connection references" in rem, rem - assert "make.powerapps.com/environments/env-deeplinks/solutions" in rem, rem - - -def test_env_004_unbound_conn_remediation_names_connector_and_connection(monkeypatch): - """The verification steps reference both the connection's display - name (so operators can spot it in the connections list) AND the - connector API id (so the Power Automate verification step is - actionable — flows are filtered by connector). Dropping either - forces the operator to cross-reference the original result text.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("orphan-conn", display_name="My Workday Conn")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: []) - - uc = next( - r for r in env_mod._check_connections_and_refs(runner) - if r.checkpoint_id.startswith("ENV-004-UC-") - ) - rem = uc.remediation or "" - assert "My Workday Conn" in rem, rem - # The connector label comes from the apiId in the _conn fixture. - assert "shared_workdaysoap" in rem, rem - - -# --------------------------------------------------------------------------- -# Walkthrough prose + doc_link pins -# -# The Solutions deep link by itself is not enough — Connection References -# are only visible *inside* a solution, under Objects → Connection -# references. The remediations must spell that out, and the doc_link -# must point to Microsoft's canonical walkthrough for binding a -# connection reference, so an operator unfamiliar with the maker portal -# has a complete reference next to the abbreviated steps. -# --------------------------------------------------------------------------- - -_CONN_REF_DOC = ( - "https://learn.microsoft.com/en-us/power-apps/maker/" - "data-platform/create-connection-reference" -) - - -def test_env_004_orphan_detail_prose_calls_out_objects_pane(monkeypatch): - """Operators reported they couldn't find Connection References from - the Solutions list. The remediation must explicitly walk them - through `Objects → Connection references` inside the solution.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("missing-conn-id", display="Workday")]) - - results = env_mod._check_connections_and_refs(runner) - - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - rem = orphan.remediation or "" - assert "Objects" in rem and "Connection references" in rem, rem - - -def test_env_004_unbound_ref_detail_prose_calls_out_objects_pane(monkeypatch): - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("", display="UnboundRef")]) - - results = env_mod._check_connections_and_refs(runner) - - unbound = next(r for r in results if r.checkpoint_id.startswith("ENV-004-UR-")) - rem = unbound.remediation or "" - assert "Objects" in rem and "Connection references" in rem, rem - - -def test_env_004_summary_prose_calls_out_objects_pane_when_failed(monkeypatch): - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("missing-conn-id", display="Workday")]) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - rem = summary.remediation or "" - assert "Objects" in rem and "Connection references" in rem, rem - - -def test_env_004_failed_summary_doc_link_points_to_connection_reference_doc(monkeypatch): - """When the summary is FAILED (ref problems), surface Microsoft's - canonical connection-reference walkthrough — the generic - ess-prepare doc doesn't help an operator fix a binding.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("missing-conn-id", display="Workday")]) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - assert summary.status == "Failed" - assert summary.doc_link == _CONN_REF_DOC - - -def test_env_004_warning_only_summary_keeps_generic_doc_link(monkeypatch): - """A WARNING-only summary (unbound connections, no orphan refs) is - not really about connection references, so keep the generic - ess-prepare doc as the summary's doc_link.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("bound-conn"), _conn("unbound-conn")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("bound-conn")]) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - assert summary.status == "Warning" - assert summary.doc_link != _CONN_REF_DOC - assert "prepare" in (summary.doc_link or "") - - -def test_env_004_orphan_detail_doc_link_points_to_connection_reference_doc(monkeypatch): - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("missing-conn-id", display="Workday")]) - - results = env_mod._check_connections_and_refs(runner) - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - assert orphan.doc_link == _CONN_REF_DOC +def _unbound(connector="shared_service-now"): + return ab.connection_reference_change(connector=connector, connection_id=None) -def test_env_004_unbound_ref_detail_doc_link_points_to_connection_reference_doc(monkeypatch): - from flightcheck.checks import environment as env_mod - runner = _make_runner(connections=[]) - monkeypatch.setattr(env_mod, "query_all", lambda *a, **kw: [_ref("", display="UnboundRef")]) +# ───────────────────────────────────────────────────────────────────── +# Connection-reference classification. +# ───────────────────────────────────────────────────────────────────── - results = env_mod._check_connections_and_refs(runner) - unbound = next(r for r in results if r.checkpoint_id.startswith("ENV-004-UR-")) - assert unbound.doc_link == _CONN_REF_DOC - - -# --------------------------------------------------------------------------- -# Per-row specific-solution deep links -# -# The env-wide solutions list is unhelpful — it dumps every first-party -# and ISV solution on the operator and leaves them guessing which one -# holds the broken ref. ENV-004 now resolves each ref's `_solutionid_value` -# to a friendly solution name + GUID and emits a deep link straight to -# that solution's detail page, where the Objects → Connection references -# pane lives. -# --------------------------------------------------------------------------- - -_SOL_ID = "11111111-2222-3333-4444-555555555555" -_SOL_URL_FRAGMENT = f"/solutions/{_SOL_ID}" - - -def test_env_004_orphan_detail_deep_links_to_specific_solution(monkeypatch): - """When we can resolve the ref's containing solution, the orphan - detail row must deep-link to that specific solution (not the - env-wide solutions list).""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - _patch_query_all( - monkeypatch, env_mod, - conn_refs=[_ref("missing-conn-id", display="Workday", solution_id=_SOL_ID)], - solutions=[_solution(_SOL_ID, "Workday Agent Solution")], - ) - - results = env_mod._check_connections_and_refs(runner) - - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - rem = orphan.remediation or "" - assert _SOL_URL_FRAGMENT in rem, rem - # The link text should surface the friendly solution name so the - # operator can confirm they're opening the right one. - assert "Workday Agent Solution" in rem, rem +class TestConnectionReferences: + def test_all_bound_passes(self): + components = ab.components_with_references( + references=[_bound(), _bound(connector="shared_workdaysoap")] + ) + runner = _runner(agentbuilder=_FakeAgentBuilder(components=components)) + summary = _by_id(_check(runner))["ENV-004"] -def test_env_004_unbound_ref_detail_deep_links_to_specific_solution(monkeypatch): - from flightcheck.checks import environment as env_mod + assert summary.status == "Passed" + assert "2 reference(s) declared by the agent(s)" in summary.result + assert "2 bound" in summary.result + assert "unbound" not in summary.result - runner = _make_runner(connections=[]) - _patch_query_all( - monkeypatch, env_mod, - conn_refs=[_ref("", display="UnboundRef", solution_id=_SOL_ID)], - solutions=[_solution(_SOL_ID, "Workday Agent Solution")], - ) + def test_unbound_ref_fails_with_detail_row(self): + components = ab.components_with_references( + references=[_bound(), _unbound(connector="shared_workdaysoap")] + ) + runner = _runner(agentbuilder=_FakeAgentBuilder(components=components)) + + results = _check(runner) + by_id = _by_id(results) + summary = by_id["ENV-004"] + + assert summary.status == "Failed" + assert "1 unbound" in summary.result + # Detail row names the offending ref and points at the Objects pane. + detail = by_id["ENV-004-UR-001"] + assert detail.status == "Failed" + assert "shared_workdaysoap" in detail.description + assert "No connection bound to this reference" in detail.result + assert "Connection references" in detail.remediation + assert "bind" in detail.remediation.lower() + + def test_failed_summary_deep_links_to_solutions_and_ref_doc(self): + components = ab.components_with_references(references=[_unbound()]) + runner = _runner(agentbuilder=_FakeAgentBuilder(components=components)) + + summary = _by_id(_check(runner))["ENV-004"] + + assert summary.status == "Failed" + assert ( + "make.powerapps.com/environments/env-deeplinks/solutions" + in summary.remediation + ) + assert "create-connection-reference" in summary.doc_link - results = env_mod._check_connections_and_refs(runner) + def test_unbound_without_env_id_falls_back_to_prose(self): + components = ab.components_with_references(references=[_unbound()]) + runner = _runner( + agentbuilder=_FakeAgentBuilder(components=components), env_id=None + ) - unbound = next(r for r in results if r.checkpoint_id.startswith("ENV-004-UR-")) - rem = unbound.remediation or "" - assert _SOL_URL_FRAGMENT in rem, rem - assert "Workday Agent Solution" in rem, rem + summary = _by_id(_check(runner))["ENV-004"] + assert summary.status == "Failed" + assert "make.powerapps.com" not in summary.remediation + assert "Connection references" in summary.remediation -def test_env_004_detail_falls_back_to_env_solutions_when_lookup_fails(monkeypatch): - """If the solutions lookup returns nothing (Dataverse error, missing - permission, the ref isn't in any solutioncomponent row), the - remediation must fall back to the env-wide solutions URL — never - produce a 404 or a half-formed markdown link.""" - from flightcheck.checks import environment as env_mod + def test_no_agentbuilder_client_skips(self): + summary = _by_id(_check(_runner(agentbuilder=None)))["ENV-004"] - runner = _make_runner(connections=[_conn("real-conn-id")]) - # Conn-ref query succeeds, but solutioncomponents returns nothing - # for this ref — simulates a permission/visibility gap mid-check. - _patch_query_all( - monkeypatch, env_mod, - conn_refs=[_ref("missing-conn-id", display="Workday", solution_id=_SOL_ID)], - solutions=[], # also empty for good measure - ) - # Override the solutioncomponents leg to return nothing. - real_fake = env_mod.query_all - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "solutioncomponents": - return [] - return real_fake(env_url, token, entity_set, select, filter_expr=filter_expr) - - monkeypatch.setattr(env_mod, "query_all", _fake) - - results = env_mod._check_connections_and_refs(runner) - - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - rem = orphan.remediation or "" - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in rem - assert _SOL_URL_FRAGMENT not in rem # specific solution should NOT appear - # The fallback link text reverts to the generic label. - assert "Power Apps \u2192 Solutions" in rem - - -def test_env_004_detail_falls_back_when_dataverse_solutions_query_raises(monkeypatch): - """A Dataverse exception during the lookup must not abort ENV-004 — - it should silently fall back to the env-wide URL so the operator - still gets actionable remediation. Covers both the - solutioncomponents and solutions queries failing.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - return [{ - "connectionreferenceid": "broken-ref-id", - "connectionreferencelogicalname": "logical_name", - "connectionreferencedisplayname": "Workday", - "connectorid": "shared_workdaysoap", - "connectionid": "missing-conn-id", - "statuscode": 1, - }] - if entity_set == "solutioncomponents": - raise RuntimeError("403 Forbidden") - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - - results = env_mod._check_connections_and_refs(runner) - - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - rem = orphan.remediation or "" - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in rem - assert _SOL_URL_FRAGMENT not in rem - - -def test_env_004_solution_lookup_uses_distinct_solution_ids(monkeypatch): - """When multiple refs live in the same solution, the lookup must - de-dupe so we don't build an enormous OData filter and don't issue - multiple round-trips.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - sol_filters: list[str] = [] - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - return [ - {"connectionreferenceid": "r1", "connectionreferencelogicalname": "n", - "connectionreferencedisplayname": "Ref1", "connectorid": "x", - "connectionid": "missing-1", "statuscode": 1}, - {"connectionreferenceid": "r2", "connectionreferencelogicalname": "n", - "connectionreferencedisplayname": "Ref2", "connectorid": "x", - "connectionid": "missing-2", "statuscode": 1}, - {"connectionreferenceid": "r3", "connectionreferencelogicalname": "n", - "connectionreferencedisplayname": "Ref3", "connectorid": "x", - "connectionid": "missing-3", "statuscode": 1}, - ] - if entity_set == "solutioncomponents": - # All 3 refs live in the same solution. - return [ - {"objectid": rid, "_solutionid_value": _SOL_ID} - for rid in ("r1", "r2", "r3") - ] - if entity_set == "solutions": - sol_filters.append(filter_expr or "") - return [_solution(_SOL_ID, "Single Solution")] - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - - env_mod._check_connections_and_refs(runner) - - # Exactly one solutions query, with exactly one solutionid filter. - assert len(sol_filters) == 1, sol_filters - assert sol_filters[0].count("solutionid eq") == 1, sol_filters[0] - - -def test_env_004_resolves_multiple_distinct_solutions(monkeypatch): - """When two broken refs live in two different solutions, each - detail row must deep-link to its own solution.""" - from flightcheck.checks import environment as env_mod - - sol_a = "aaaaaaaa-0000-0000-0000-000000000001" - sol_b = "bbbbbbbb-0000-0000-0000-000000000002" - - runner = _make_runner(connections=[_conn("real-conn-id")]) - _patch_query_all( - monkeypatch, env_mod, - conn_refs=[ - _ref("missing-1", ref_id="r-a", display="RefA", solution_id=sol_a), - _ref("missing-2", ref_id="r-b", display="RefB", solution_id=sol_b), - ], - solutions=[ - _solution(sol_a, "Solution A"), - _solution(sol_b, "Solution B"), - ], - ) + assert summary.status == "Skipped" + assert "AgentBuilder client or agent botId not available" in summary.result + assert "AgentBuilder" in summary.remediation - results = env_mod._check_connections_and_refs(runner) + def test_no_configured_botid_skips(self): + runner = _runner( + agentbuilder=_FakeAgentBuilder(), config={} + ) + summary = _by_id(_check(runner))["ENV-004"] - orphan_rows = [r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")] - assert len(orphan_rows) == 2 + assert summary.status == "Skipped" + assert "not available" in summary.result - by_ref = {r.description: r.remediation or "" for r in orphan_rows} - assert f"/solutions/{sol_a}" in by_ref["Orphan reference: RefA"] - assert "Solution A" in by_ref["Orphan reference: RefA"] - assert f"/solutions/{sol_b}" in by_ref["Orphan reference: RefB"] - assert "Solution B" in by_ref["Orphan reference: RefB"] + def test_malformed_changeset_degrades_to_warning(self): + runner = _runner( + agentbuilder=_FakeAgentBuilder( + components={"connectionReferenceChanges": {"unexpected": "dict"}} + ) + ) + summary = _by_id(_check(runner))["ENV-004"] + + assert summary.status == "Warning" + assert "Unable to read the agent's connection references" in summary.result + assert "ValueError" in summary.result + assert "AgentBuilder" in summary.remediation + + def test_references_unioned_across_configured_agents(self): + # Two agents, each declaring the SAME logical name — the union must + # de-dupe so the shared ref is judged once, not twice. + shared = ab.connection_reference_change( + connector="shared_service-now", + connection_id=None, + logical_name="gptagent_ess.shared_ref", + ) + class _TwoAgent(_FakeAgentBuilder): + def fetch_components(self, _agent_id): + return ab.components_with_references(references=[shared]) + + runner = _runner( + agentbuilder=_TwoAgent(), + config={ + "agents": [ + {"botId": "bot-a"}, + {"botId": "bot-b"}, + ] + }, + ) + results = _check(runner) + by_id = _by_id(results) + + assert by_id["ENV-004"].result.startswith("1 reference(s)") + assert "ENV-004-UR-001" in by_id + assert "ENV-004-UR-002" not in by_id + + +# ───────────────────────────────────────────────────────────────────── +# GRS commit pin (ENV-004-GRS). +# ───────────────────────────────────────────────────────────────────── + + +class TestGrsCommitPin: + def test_pin_skipped_and_omitted_from_summary_when_no_expected_sha(self): + components = ab.components_with_references(references=[_bound()]) + runner = _runner(agentbuilder=_FakeAgentBuilder(components=components)) + + results = _check(runner) + by_id = _by_id(results) + + assert by_id["ENV-004-GRS"].status == "Skipped" + assert "No expected GRS commit SHA is configured" in by_id["ENV-004-GRS"].result + # A SKIPPED pin does not appear in the summary and does not change PASS. + assert by_id["ENV-004"].status == "Passed" + assert "GRS commit pin" not in by_id["ENV-004"].result + + def test_matching_commit_passes_and_folds_into_summary(self): + components = ab.components_with_references(references=[_bound()]) + runner = _runner( + agentbuilder=_FakeAgentBuilder( + components=components, configuration=ab.configuration() + ), + config={ + "agent": {"botId": ab.MOCK_AGENT_ID}, + "expectedGrsCommitSha": ab.COMMIT_SHA, + }, + ) + by_id = _by_id(_check(runner)) + + assert by_id["ENV-004-GRS"].status == "Passed" + assert ab.COMMIT_SHA in by_id["ENV-004-GRS"].result + assert by_id["ENV-004"].status == "Passed" + assert "GRS commit pin: Passed" in by_id["ENV-004"].result + + def test_mismatched_commit_fails_and_forces_summary_fail(self): + components = ab.components_with_references(references=[_bound()]) + runner = _runner( + agentbuilder=_FakeAgentBuilder( + components=components, + configuration=ab.configuration(commit_sha="deadbeef"), + ), + config={ + "agent": {"botId": ab.MOCK_AGENT_ID}, + "expectedGrsCommitSha": ab.COMMIT_SHA, + }, + ) + by_id = _by_id(_check(runner)) + + grs = by_id["ENV-004-GRS"] + assert grs.status == "Failed" + assert "deadbeef" in grs.result + assert "Publish or import the ESS agent solution" in grs.remediation + # Even though every reference is bound, the GRS mismatch fails ENV-004. + assert by_id["ENV-004"].status == "Failed" + assert "GRS commit pin: Failed" in by_id["ENV-004"].result + + def test_missing_commit_in_configure_fails(self): + components = ab.components_with_references(references=[_bound()]) + runner = _runner( + agentbuilder=_FakeAgentBuilder( + components=components, + configuration=ab.configuration(commit_sha=""), + ), + config={ + "agent": {"botId": ab.MOCK_AGENT_ID}, + "expectedGrsCommitSha": ab.COMMIT_SHA, + }, + ) + grs = _by_id(_check(runner))["ENV-004-GRS"] + + assert grs.status == "Failed" + assert "returned no commitSha" in grs.result + assert "Publish or import the ESS agent solution" in grs.remediation + + def test_configure_read_error_warns(self): + components = ab.components_with_references(references=[_bound()]) + runner = _runner( + agentbuilder=_FakeAgentBuilder( + components=components, + configure_error=RuntimeError("boom"), + ), + config={ + "agent": {"botId": ab.MOCK_AGENT_ID}, + "expectedGrsCommitSha": ab.COMMIT_SHA, + }, + ) + by_id = _by_id(_check(runner)) + + grs = by_id["ENV-004-GRS"] + assert grs.status == "Warning" + assert "Could not read minimalBots ALM configure" in grs.result + assert "RuntimeError: boom" in grs.result + assert "signed in to Copilot Studio" in grs.remediation + # A GRS WARNING folds into the summary as a WARNING (refs all bound). + assert by_id["ENV-004"].status == "Warning" + assert "GRS commit pin: Warning" in by_id["ENV-004"].result + + def test_invalid_realm_fails(self): + components = ab.components_with_references(references=[_bound()]) + runner = _runner( + agentbuilder=_FakeAgentBuilder( + components=components, configuration=ab.configuration() + ), + config={ + "agent": {"botId": ab.MOCK_AGENT_ID}, + "expectedGrsCommitSha": ab.COMMIT_SHA, + "grsRealm": "Staging", + }, + ) + grs = _by_id(_check(runner))["ENV-004-GRS"] + + assert grs.status == "Failed" + assert "'Staging' is invalid" in grs.result + assert "Dev, Test, or Prod" in grs.remediation + + def test_dev_realm_zero_is_valid(self): + # Regression guard: realm Dev maps to the int 0, which is falsy; the + # check must treat 0 as a valid realm, not as "unconfigured". + components = ab.components_with_references(references=[_bound()]) + runner = _runner( + agentbuilder=_FakeAgentBuilder( + components=components, configuration=ab.configuration() + ), + config={ + "agent": {"botId": ab.MOCK_AGENT_ID}, + "expectedGrsCommitSha": ab.COMMIT_SHA, + "grsRealm": "Dev", + }, + ) + grs = _by_id(_check(runner))["ENV-004-GRS"] -def test_resolve_ref_solutions_no_refs_skips_dataverse_call(monkeypatch): - """If there are no problematic refs, we must not issue an empty - OData filter (`objectid eq` with nothing) — skip both round-trips.""" - from flightcheck.checks import environment as env_mod + assert grs.status == "Passed" + assert "realm Dev" in grs.result - called = [] - def _fake(env_url, token, entity_set, select, filter_expr=None): - called.append(entity_set) - return [] +# ───────────────────────────────────────────────────────────────────── +# Maker URL builders (generic helpers ENV-004 remediations rely on). +# ───────────────────────────────────────────────────────────────────── - monkeypatch.setattr(env_mod, "query_all", _fake) - out = env_mod._resolve_ref_solutions( - env_url="https://x.api.crm.dynamics.com", - dv_token="tok", - env_id="env-1", - refs=[], - ) - assert out == {} - assert "solutioncomponents" not in called - assert "solutions" not in called - - -# --------------------------------------------------------------------------- -# Summary remediation when broken refs resolve to specific solutions -# -# Pin the smarter summary text: -# - All broken refs in ONE solution → summary deep-links to that solution -# - Broken refs SPREAD across solutions → summary names them and points -# at the env-wide list -# - Lookup failed → summary keeps the generic prose (don't pretend to -# know something we don't) -# --------------------------------------------------------------------------- - -def test_env_004_summary_deep_links_when_all_refs_in_one_solution(monkeypatch): - """When every broken ref lives in the same solution, the summary - must deep-link to that specific solution — not the env-wide list.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - _patch_query_all( - monkeypatch, env_mod, - conn_refs=[ - _ref("missing-1", ref_id="r1", display="RefA", solution_id=_SOL_ID), - _ref("missing-2", ref_id="r2", display="RefB", solution_id=_SOL_ID), - ], - solutions=[_solution(_SOL_ID, "Workday Agent Solution")], - ) +def test_maker_solutions_url_targets_powerapps(): + from flightcheck.checks._maker_urls import maker_solutions_url - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - rem = summary.remediation or "" - assert _SOL_URL_FRAGMENT in rem, rem - assert "Workday Agent Solution" in rem, rem - # The "click the solution that contains your agent" hedge wording - # should be gone now that we know the exact solution. - assert "click the solution that contains your agent" not in rem - - -def test_env_004_summary_lists_solutions_when_refs_spread(monkeypatch): - """When broken refs span multiple solutions, the summary must name - every affected solution so the operator knows where to look.""" - from flightcheck.checks import environment as env_mod - - sol_a = "aaaaaaaa-0000-0000-0000-000000000001" - sol_b = "bbbbbbbb-0000-0000-0000-000000000002" - - runner = _make_runner(connections=[_conn("real-conn-id")]) - _patch_query_all( - monkeypatch, env_mod, - conn_refs=[ - _ref("missing-1", ref_id="r-a", display="RefA", solution_id=sol_a), - _ref("missing-2", ref_id="r-b", display="RefB", solution_id=sol_b), - ], - solutions=[ - _solution(sol_a, "Workday Agent Solution"), - _solution(sol_b, "Custom Tweaks Solution"), - ], + assert maker_solutions_url("env-123") == ( + "https://make.powerapps.com/environments/env-123/solutions" ) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - rem = summary.remediation or "" - # Both affected solution names must be surfaced. - assert "Workday Agent Solution" in rem, rem - assert "Custom Tweaks Solution" in rem, rem - # The env-wide solutions URL is the only viable link target when - # multiple solutions are involved. - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in rem - # And it must NOT deep-link to either specific solution (would be - # misleading — there are TWO to visit). - assert f"/solutions/{sol_a}" not in rem - assert f"/solutions/{sol_b}" not in rem - - -def test_env_004_summary_keeps_generic_prose_when_lookup_unresolved(monkeypatch): - """If the solutioncomponents lookup returns nothing for any ref, - the summary must fall back to the generic prose — don't claim to - know solutions we couldn't resolve.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - return [{ - "connectionreferenceid": "broken-ref", - "connectionreferencelogicalname": "n", - "connectionreferencedisplayname": "Workday", - "connectorid": "x", - "connectionid": "missing-conn", - "statuscode": 1, - }] - if entity_set == "solutioncomponents": - return [] # ref not found in any solution component - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - - results = env_mod._check_connections_and_refs(runner) - - summary = next(r for r in results if r.checkpoint_id == "ENV-004") - rem = summary.remediation or "" - # Generic prose hallmark — the hedge wording. - assert "click the solution that contains your agent" in rem - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in rem - -# --------------------------------------------------------------------------- -# Managed-solution fallback to Default Solution. -# -# Power Apps blocks direct edits inside managed solutions with -# "You cannot directly edit the objects within a managed solution.", -# so a remediation link that lands on one is a dead end for the maker. -# The check redirects the link to Default Solution (always unmanaged, -# always present) so the maker has somewhere to actually re-bind. -# --------------------------------------------------------------------------- - - -_DEFAULT_SOL_ID = "00000000-0000-0000-0000-00000000DEFA" -_DEFAULT_SOL_URL_FRAGMENT = f"/solutions/{_DEFAULT_SOL_ID}" - - -def test_env_004_managed_only_ref_links_to_default_solution(monkeypatch): - """When a broken ref only lives in a *managed* solution, the link - must redirect to the Default Solution (the unmanaged customization - layer) — never to a managed solution where edits are blocked.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - managed_sid = "00000000-0000-0000-0000-0000MANAGED1" - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - return [_ref("missing-conn-id", display="Workday")] - if entity_set == "solutioncomponents": - return [{"objectid": "ref-id", "_solutionid_value": managed_sid}] - if entity_set == "solutions": - return [ - _solution(managed_sid, "Workday Managed", ismanaged=True), - _solution(_DEFAULT_SOL_ID, "Default Solution", - unique_name="Default", ismanaged=False), - ] - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - - results = env_mod._check_connections_and_refs(runner) - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - rem = orphan.remediation or "" - # Link goes to Default, NOT to the managed solution. - assert _DEFAULT_SOL_URL_FRAGMENT in rem, rem - assert f"/solutions/{managed_sid}" not in rem, rem - # Label should reflect the actual destination so the maker isn't - # surprised by what they see when they click. - assert "Default Solution" in rem, rem - - -def test_env_004_prefers_named_unmanaged_over_default_solution(monkeypatch): - """If a ref lives in BOTH a named unmanaged solution and Default, - prefer the named one — it's the maker's focused workspace, while - Default is the catch-all that includes every component in the org.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - named_sid = "00000000-0000-0000-0000-0000NAMED0001" - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - return [_ref("missing-conn-id", display="Workday")] - if entity_set == "solutioncomponents": - # Same ref appears in two solutions. - return [ - {"objectid": "ref-id", "_solutionid_value": named_sid}, - {"objectid": "ref-id", "_solutionid_value": _DEFAULT_SOL_ID}, - ] - if entity_set == "solutions": - return [ - _solution(named_sid, "Workday Customizations", - unique_name="WorkdayCustom", ismanaged=False), - _solution(_DEFAULT_SOL_ID, "Default Solution", - unique_name="Default", ismanaged=False), - ] - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - - results = env_mod._check_connections_and_refs(runner) - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - rem = orphan.remediation or "" - assert f"/solutions/{named_sid}" in rem, rem - assert _DEFAULT_SOL_URL_FRAGMENT not in rem, rem - assert "Workday Customizations" in rem, rem - - -def test_env_004_managed_only_with_no_default_falls_back_to_env_wide(monkeypatch): - """Edge case: ref only lives in a managed solution AND Dataverse - didn't return Default (shouldn't happen in practice — Default is - always present — but be defensive). Must not link to the managed - solution; fall back to the env-wide URL instead.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - managed_sid = "00000000-0000-0000-0000-0000MANAGED2" - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - return [_ref("missing-conn-id", display="Workday")] - if entity_set == "solutioncomponents": - return [{"objectid": "ref-id", "_solutionid_value": managed_sid}] - if entity_set == "solutions": - # No Default returned — only the managed solution. - return [_solution(managed_sid, "Managed Only", ismanaged=True)] - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - - results = env_mod._check_connections_and_refs(runner) - orphan = next(r for r in results if r.checkpoint_id.startswith("ENV-004-OR-")) - rem = orphan.remediation or "" - # Never link to the managed solution. - assert f"/solutions/{managed_sid}" not in rem, rem - # Fall back to the env-wide solutions URL. - assert "https://make.powerapps.com/environments/env-deeplinks/solutions" in rem - # And no other specific /solutions// fragment leaked through. - assert "/solutions/00000000" not in rem, rem - - -def test_env_004_solutions_filter_always_includes_default_uniquename(monkeypatch): - """The solutions OData filter must always include - `uniquename eq 'Default'` so the Default Solution row comes back - even when the ref's containing solutions don't intersect it. Without - this, the managed-only fallback has nowhere to redirect.""" - from flightcheck.checks import environment as env_mod - - runner = _make_runner(connections=[_conn("real-conn-id")]) - sol_filters: list[str] = [] - - def _fake(env_url, token, entity_set, select, filter_expr=None): - if entity_set == "connectionreferences": - return [_ref("missing-conn-id", display="Workday")] - if entity_set == "solutioncomponents": - return [{"objectid": "ref-id", "_solutionid_value": _SOL_ID}] - if entity_set == "solutions": - sol_filters.append(filter_expr or "") - return [_solution(_SOL_ID, "Some Solution")] - return [] - - monkeypatch.setattr(env_mod, "query_all", _fake) - - env_mod._check_connections_and_refs(runner) - - assert len(sol_filters) == 1, sol_filters - assert "uniquename eq 'Default'" in sol_filters[0], sol_filters[0] diff --git a/tests/flightcheck/checks/test_workday_extension.py b/tests/flightcheck/checks/test_workday_extension.py index dbb97f857..8983cc9ec 100644 --- a/tests/flightcheck/checks/test_workday_extension.py +++ b/tests/flightcheck/checks/test_workday_extension.py @@ -10,9 +10,9 @@ connection, degrades gracefully when it does not. Cached-ref read + a best-effort Power Platform admin owner echo — no cassette required (the admin connections listing is the ``validated`` pp_admin mock). - * DV-CONN-001 — PASS/FAIL/NOT_CONFIGURED/SKIPPED over a documented-tier - Dataverse ``connectionreferences`` read (stubbed with ``responses``); owner - echo via the ``validated`` pp_admin mock. + * DV-CONN-001 — PASS/FAIL/SKIPPED over the validated minimalBots components + read (Workday SOAP connection reference; faked ``runner.agentbuilder``); + owner echo via the ``validated`` pp_admin mock. * WD-REST-001 — pure-config check (restBaseUrl trimmed to '/api'). * WD-REST-002 — pure local-file check (user-context redirect topic); SKIPPED on the legacy install path. @@ -28,22 +28,18 @@ from dataclasses import dataclass, field from typing import Any -import responses - from tests.conftest import require_validated_mock +from tests.mocks import agentbuilder_connectivity as ab from tests.mocks import dataverse as dv from tests.mocks import pp_admin as pp +require_validated_mock(ab) require_validated_mock(dv) require_validated_mock(pp) from flightcheck.checks import workday_extension as wx # noqa: E402 from flightcheck.runner import Priority, Role, Status # noqa: E402 -_DV_CONNECTOR_ID = ( - "/providers/Microsoft.PowerApps/apis/shared_commondataserviceforapps" -) - # ───────────────────────────────────────────────────────────────────── # Minimal runner. The emitters read only these attributes; anything the @@ -62,6 +58,17 @@ def get_connections(self, _env_id: str): return self._connections +class _FakeAgentBuilder: + """Stand-in for FlightCheckRunner.agentbuilder. Only ``fetch_components`` + is consumed (DV-CONN-001's connection-reference read).""" + + def __init__(self, components: dict[str, Any]): + self._components = components + + def fetch_components(self, _agent_id: str): + return self._components + + @dataclass class _Runner: config: Any = field(default_factory=dict) @@ -69,6 +76,7 @@ class _Runner: env_url: str | None = None dv_token: str | None = None pp_admin: Any = None + agentbuilder: Any = None env_id: str | None = None agent_slug: str = "" _workday_connection_refs: list[dict[str, Any]] = field(default_factory=list) @@ -90,28 +98,6 @@ def _by_id(results): return {r.checkpoint_id: r for r in results} -def _dv_ref(*, connection_id, statuscode=1): - """A Dataverse connection reference matching the extension pack's shipped - ref (connector shared_commondataserviceforapps, logical-name suffix - 92b66).""" - return dv.connection_ref( - logical_name="msdyn_sharedcommondataserviceforapps_92b66", - display_name="Microsoft Dataverse", - connector_id=_DV_CONNECTOR_ID, - connection_id=connection_id, - statuscode=statuscode, - ) - - -def _register_refs(base_url: str, refs: list[dict[str, Any]]) -> None: - responses.add( - method="GET", - url=f"{base_url}/api/data/v9.2/connectionreferences", - json=dv.collection(refs), - status=200, - ) - - # ───────────────────────────────────────────────────────────────────── # WD-CONN-AUTH-001 — always MANUAL echo (S5.3). # ───────────────────────────────────────────────────────────────────── @@ -255,147 +241,111 @@ def test_never_passes_regardless_of_state(self): # ───────────────────────────────────────────────────────────────────── -# DV-CONN-001 — Dataverse connection binding (S5.4, PASS/FAIL). +# DV-CONN-001 — Workday SOAP connection binding (S5.4, PASS/FAIL). # ───────────────────────────────────────────────────────────────────── +def _runner_with_refs(references, *, pp_admin=None, env_id=None): + """A runner whose faked ``agentbuilder.fetch_components`` returns the given + connection references and whose config names an active agent (botId).""" + components = ab.components_with_references(references=references) + return _Runner( + config={"agent": {"botId": ab.MOCK_AGENT_ID}}, + agentbuilder=_FakeAgentBuilder(components), + pp_admin=pp_admin, + env_id=env_id, + ) + + class TestDataverseConnection: - @responses.activate - def test_bound_active_with_owner_echo_passes( - self, fake_dataverse_url, fake_token - ): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-active", statuscode=1)], - ) + def test_bound_with_owner_echo_passes(self): owner_conn = pp.connection( - name="dv-conn-active", - api_name="shared_commondataserviceforapps", + name="wd-conn-active", + api_name="shared_workdaysoap", extra_properties={"accountName": "maker@contoso.com"}, ) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id="wd-conn-active")], pp_admin=_FakePPAdmin([owner_conn]), env_id="env-1", ) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.PASSED.value - assert "bound to an active" in r.result + assert "bound to a connection" in r.result assert "maker@contoso.com" in r.result assert "your own account" in r.result - @responses.activate - def test_passes_without_pp_admin_notes_owner_unreadable( - self, fake_dataverse_url, fake_token - ): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-active", statuscode=1)], + def test_passes_without_pp_admin_notes_owner_unreadable(self): + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id="wd-conn-active")], ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.PASSED.value assert "owner could not be read" in r.result assert "your own account" in r.result - @responses.activate - def test_runtime_dataverse_reference_passes( - self, fake_dataverse_url, fake_token - ): - runtime_ref = dv.workday_connection_refs_runtime()[1] - _register_refs(fake_dataverse_url, [runtime_ref]) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, + def test_unbound_fails(self): + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id=None)], ) - - r = _by_id( - wx.run_workday_extension_checks(runner) - )["DV-CONN-001"] - - assert r.status == Status.PASSED.value - assert ( - "msdyn_sharedcommondataserviceforapps_workdayruntime" - in r.result - ) - - @responses.activate - def test_mixed_runtime_and_legacy_dataverse_refs_warn( - self, fake_dataverse_url, fake_token - ): - runtime_ref = dv.workday_connection_refs_runtime()[1] - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="legacy-dv"), runtime_ref], - ) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, - ) - r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.WARNING.value - assert "Multiple ESS Dataverse connection references" in r.result - assert "Remove obsolete Workday package references" in r.remediation + assert r.status == Status.FAILED.value + assert "unbound" in r.result + assert "connectionId=null" in r.result + assert "bind the Workday SOAP connection reference" in r.remediation - @responses.activate - def test_unbound_fails(self, fake_dataverse_url, fake_token): - _register_refs( - fake_dataverse_url, [_dv_ref(connection_id=None, statuscode=1)] + def test_workday_ref_absent_fails(self): + # Only a ServiceNow ref is present - no Workday SOAP ref. + runner = _runner_with_refs( + [ + ab.connection_reference_change( + connector="shared_service-now", + connection_id="sn-1", + ) + ] ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.FAILED.value - assert "unbound" in r.result - assert "connectionid=null" in r.result - assert "bind the Dataverse connection reference" in r.remediation + assert "was not found" in r.result + assert "shared_workdaysoap" in r.result + assert "Install or repair the Workday extension pack" in r.remediation - @responses.activate - def test_inactive_statuscode_fails(self, fake_dataverse_url, fake_token): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-inactive", statuscode=2)], - ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) + def test_no_agentbuilder_client_skips(self): + runner = _Runner(config={"agent": {"botId": ab.MOCK_AGENT_ID}}) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.FAILED.value - assert "inactive" in r.result - assert "statuscode=2" in r.result - assert "Re-authenticate or re-bind" in r.remediation - - @responses.activate - def test_missing_ref_not_configured(self, fake_dataverse_url, fake_token): - # Only a Workday ref present — no Dataverse (92b66) ref. - _register_refs( - fake_dataverse_url, - [ - dv.connection_ref( - logical_name="new_sharedworkdaysoap_ff0df", - display_name="OAuthUser", - connector_id=dv.WORKDAY_SOAP_CONNECTOR_ID, - connection_id="wd-conn-1", - ) - ], + assert r.status == Status.SKIPPED.value + assert "not available" in r.result + + def test_no_active_agent_botid_skips(self): + runner = _Runner( + config={}, + agentbuilder=_FakeAgentBuilder(ab.components_with_references()), ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.NOT_CONFIGURED.value - assert "was not found in this environment" in r.result - assert "Install/repair the Workday extension pack" in r.remediation + assert r.status == Status.SKIPPED.value + assert "not available" in r.result - def test_no_dv_token_skips(self): - runner = _Runner(env_url="https://x.crm.dynamics.com", dv_token="") + def test_malformed_changeset_degrades_to_warning(self): + # A 200 payload whose connectionReferenceChanges is present but not a + # list is a shape we do not understand: fail loudly (dispatcher WARNING) + # rather than reporting a confident "reference not found" FAILED. + runner = _Runner( + config={"agent": {"botId": ab.MOCK_AGENT_ID}}, + agentbuilder=_FakeAgentBuilder( + {"connectionReferenceChanges": {"unexpected": "dict"}} + ), + ) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.SKIPPED.value - assert "Dataverse token not available" in r.result + assert r.status == Status.WARNING.value + assert "Unable to run DV-CONN-001" in r.result + assert "DV-CONN-001" in r.remediation # ───────────────────────────────────────────────────────────────────── diff --git a/tests/flightcheck/test_registry.py b/tests/flightcheck/test_registry.py index e8882af57..75cb99188 100644 --- a/tests/flightcheck/test_registry.py +++ b/tests/flightcheck/test_registry.py @@ -188,6 +188,21 @@ def test_env_capacity_001_resolves_and_unions_powerplatform(self): assert plan.requires_config is False assert len(plan.ordered_fns) == 1 + def test_env_004_resolves_to_agentbuilder_without_dataverse(self): + spec = registry.resolve("ENV-004") + assert spec is not None and spec.key == "ENV-004" + assert spec.category_label == "Environment" + assert spec.clients == frozenset({registry.AGENTBUILDER}) + # ENV-004 was re-pointed off the Dataverse connectionreference table to + # the Declarative Agent minimalBots components API, so it must not + # require a Dataverse endpoint, and its detail rows must not resolve. + plan = registry.transitive_requirements("ENV-004") + assert registry.AGENTBUILDER in plan.clients + assert plan.requires_dataverse_endpoint is False + assert plan.requires_config is True + assert registry.resolve("ENV-004-GRS") is None + assert registry.resolve("ENV-004-UR-001") is None + def test_native_agent_checkpoints_use_only_native_read_clients(self): access = registry.transitive_requirements("DA-AGENT-001") assert access.clients == frozenset({registry.AGENTBUILDER}) @@ -294,7 +309,7 @@ class TestWorkdayExtensionCheckpoints: """skill-5 mints five checkpoints, all sharing checks/workday_extension.run_workday_extension_checks, category "Workday Extension". Two are always-MANUAL echoes/attestations, three are - programmatic (one Dataverse read + two pure-local).""" + programmatic (one minimalBots components read + two pure-local).""" _ALL = ( "WD-CONN-AUTH-001", @@ -329,10 +344,12 @@ def test_conn_auth_exact_beats_wd_conn_family(self): assert registry.resolve("WD-CONN-AUTH-001").key == "WD-CONN-AUTH-001" assert registry.resolve("WD-CONN-AUTH-001").is_family is False - def test_dv_conn_spec_declares_dataverse_and_pp_admin(self): + def test_dv_conn_spec_declares_agentbuilder_and_pp_admin(self): spec = registry.resolve("DV-CONN-001") - assert spec.clients == frozenset({registry.DATAVERSE, registry.PP_ADMIN}) - assert spec.requires_dataverse_endpoint is True + assert spec.clients == frozenset( + {registry.AGENTBUILDER, registry.PP_ADMIN} + ) + assert spec.requires_dataverse_endpoint is False assert spec.prereqs == () assert Role.ESS_MAKER.value in spec.roles @@ -354,7 +371,7 @@ def test_net_check_is_clientless_and_ppadmin_gated(self): def test_dv_conn_plan_unions_clients(self): plan = registry.transitive_requirements("DV-CONN-001") - assert registry.DATAVERSE in plan.clients + assert registry.AGENTBUILDER in plan.clients assert registry.PP_ADMIN in plan.clients def test_all_five_are_listable(self): diff --git a/tests/mocks/agentbuilder_connectivity.py b/tests/mocks/agentbuilder_connectivity.py index f666f928c..8b149909a 100644 --- a/tests/mocks/agentbuilder_connectivity.py +++ b/tests/mocks/agentbuilder_connectivity.py @@ -22,6 +22,9 @@ MOCK_FAMILY_ID = "00000000-0000-0000-0000-000000003333" MOCK_CONNECTION_ID = "mock-servicenow-connection" MOCK_WORKDAY_CONNECTION_ID = "mock-workday-connection" +# The GRS commit pin captured in the validated ALM configure response +# (agentbuilder_readiness.yaml line 82). +COMMIT_SHA = "4bc80d2768da5de930fd56a1f5ee815b8f9d1d3b" MOCK_AGENTBUILDER_BASE = ( "https://00000000000000000000000000000000." "0.environment.api.test.powerplatform.com" @@ -37,12 +40,21 @@ def agent() -> dict[str, Any]: } -def configuration() -> dict[str, Any]: +def configuration(*, commit_sha: str = COMMIT_SHA) -> dict[str, Any]: + """The minimalBots ALM ``configure`` response (realm Dev). + + ``commitSha`` is the GRS commit pin ``ENV-004-GRS`` reads. + + Source (validated): + tests/fixtures/cassettes/agentbuilder_readiness.yaml line 82 + (``realm``/``cdsBotId``/``schemaName``/``grsRepositoryId``/``commitSha``). + """ return { "realm": "Dev", "cdsBotId": MOCK_AGENT_ID, "schemaName": "gptagent_mockemployeeselfservice", "grsRepositoryId": MOCK_FAMILY_ID, + "commitSha": commit_sha, }