diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/_da_components.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/_da_components.py new file mode 100644 index 000000000..b097e37cb --- /dev/null +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/_da_components.py @@ -0,0 +1,64 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Helpers for reading Declarative Agent components from AgentBuilder.""" + +from __future__ import annotations + +from typing import Any + + +def agent_bot_id(config: dict[str, Any]) -> str: + """Return the active configured botId, or ``""`` when none is available.""" + active_slug = config.get("activeAgent") or (config.get("agent") or {}).get( + "slug" + ) + agents = config.get("agents") or [] + if active_slug: + for agent in agents: + if not isinstance(agent, dict): + continue + if agent.get("slug") == active_slug and agent.get("botId"): + return str(agent["botId"]) + for agent in agents: + if isinstance(agent, dict) and agent.get("botId"): + return str(agent["botId"]) + return str((config.get("agent") or {}).get("botId") or "") + + +def _component_schema_name(change: Any) -> str: + if not isinstance(change, dict): + return "" + component = change.get("component") + if not isinstance(component, dict): + component = change.get("botComponent") + if not isinstance(component, dict): + component = change + return str( + component.get("schemaName") + or component.get("name") + or change.get("schemaName") + or "" + ) + + +def read_component_schema_names(runner) -> set[str] | None: + """Return schema names from AgentBuilder ``botComponentChanges``. + + ``None`` means the AgentBuilder client or active botId is unavailable. + A missing ``botComponentChanges`` key means the agent has no component + changes. A present non-list value is an invalid API shape and raises. + """ + client = getattr(runner, "agentbuilder", None) + config = getattr(runner, "config", None) or {} + bot_id = agent_bot_id(config) + if client is None or not bot_id: + return None + + payload = client.fetch_components(bot_id) or {} + changes = payload.get("botComponentChanges") + if changes is None: + return set() + if not isinstance(changes, list): + raise ValueError("Component fetch returned invalid botComponentChanges.") + return {name for name in (_component_schema_name(c) for c in changes) if name} diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py index c229cebea..f3495a14e 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday.py @@ -38,6 +38,7 @@ from defusedxml.common import DefusedXmlException from ..runner import CheckResult, Priority, Role, Status +from ._da_components import read_component_schema_names from .. import live_egress_probe from .infrastructure import ( _infra_003_directive, @@ -367,44 +368,11 @@ # request-template ID types do NOT map 1:1 to GetReferenceData keys and would # false-positive on OOTB scenarios. Confirmed on a live tenant 2026-06.) -# Friendly labels for the result text (best-effort; unknown keys print raw). -_WD_REF_KEY_LABELS = { - "ISO_3166-1_Alpha-2_Code": "Countries (ISO alpha-2)", - "ISO_3166-1_Alpha-3_Code": "Countries (ISO alpha-3)", - "Country_Phone_Code_ID": "Country phone codes", - "Phone_Device_Type_ID": "Phone device types", - "Communication_Usage_Type_ID": "Communication usage types", - "Related_Person_Relationship_ID": "Relationship types", - "Government_ID_Type_ID": "Government ID types", - "National_ID_Type_Code": "National ID types", - "Passport_ID_Type_ID": "Passport ID types", - "Visa_ID_Type_ID": "Visa ID types", - "Marital_Status_ID": "Marital status", - "Gender_ID": "Gender", - "Ethnicity_ID": "Ethnicity", - "Language_ID": "Languages", +_WD_REF_SYSTEM_TOPICS = { + "WorkdaySystemGetReferenceData", + "WorkdaySystemRefreshReferenceData", } -# In the GetReferenceData topic's switch: referenceDataKey = "KEY" -> a SUPPORTED key. -_WD_REF_SUPPORTED_RE = re.compile(r'referenceDataKey\s*=\s*["\']([^"\']+)["\']') -# In a calling topic: referenceDataKey: KEY -> a REQUESTED key (literal, same line). -# A Power Fx expression value (starts with '=') is intentionally not matched (the -# key isn't statically known), and the GetReferenceData input declaration (no -# value on the line) is likewise not matched. -_WD_REF_REQUESTED_RE = re.compile(r'referenceDataKey:[ \t]*([A-Za-z0-9_]+)') - - -def _ref_key_label(key: str) -> str: - return _WD_REF_KEY_LABELS.get(key, key) - - -def _extract_requested_reference_keys(topic_data: str) -> set[str]: - """Reference keys a topic actually REQUESTS from GetReferenceData — the - literal ``referenceDataKey: KEY`` input it passes on each call.""" - if not topic_data: - return set() - return set(_WD_REF_REQUESTED_RE.findall(topic_data)) - def _wd_studio_link(runner) -> str: """Markdown deep-link to the active agent in Copilot Studio (its **Topics** @@ -432,153 +400,178 @@ def _wd_studio_link(runner) -> str: return "[Copilot Studio](https://copilotstudio.microsoft.com/)" -def _extract_supported_reference_keys(topic_data: str) -> set[str]: - """Reference keys the GetReferenceData topic supports (its switch).""" - if not topic_data: - return set() - return set(_WD_REF_SUPPORTED_RE.findall(topic_data)) - - -def _check_workday_reference_data(runner) -> list[CheckResult]: - """WD-REF-001 — verify every reference picklist a Workday topic requests is - supported by the shared GetReferenceData topic. - - Pure Dataverse reconciliation (``documented`` tier — no external API): - * read all topic ``botcomponents`` -> per topic, the reference keys it - REQUESTS from GetReferenceData, plus GetReferenceData's SUPPORTED keys; - * FAIL on any requested key not supported (or GetReferenceData missing). - """ - roles = [Role.ESS_MAKER.value, Role.WORKDAY_ADMIN.value] - cid = "WD-REF-001" - cat = "Workday" - doc = f"{DOC_BASE}/workday" - env_url = getattr(runner, "env_url", None) - dv_token = getattr(runner, "dv_token", None) +def _schema_leaf(schema_name: str) -> str: + return schema_name.rsplit(".", 1)[-1] - if not env_url or not dv_token: - return [CheckResult( - checkpoint_id=cid, category=cat, priority=Priority.HIGH.value, - status=Status.SKIPPED.value, - description="Workday write-scenario reference-data availability", - result="Dataverse token not available — cannot read topic configuration.", - remediation="Re-run /flightcheck with Dataverse access.", - roles=roles, - )] +def _da_component_names_or_result( + runner, + *, + checkpoint_id: str, + category: str, + description: str, + doc_link: str, + roles: list[str], +) -> tuple[set[str] | None, CheckResult | None]: + """Read AgentBuilder component schemaNames, or return a terminal result.""" try: - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - from auth import query_all - topics = query_all( - env_url, dv_token, - "botcomponents", - "name,schemaname,data", - filter_expr="componenttype eq 9", + schema_names = read_component_schema_names(runner) + except ValueError as exc: + return None, CheckResult( + checkpoint_id=checkpoint_id, + category=category, + priority=Priority.HIGH.value, + status=Status.WARNING.value, + description=description, + result=f"Unable to read Declarative Agent components: {exc}", + remediation=( + "Re-run FlightCheck. If this persists, report the checkpoint " + f"ID ({checkpoint_id}) and the invalid botComponentChanges " + "shape above." + ), + doc_link=doc_link, + roles=roles, ) - except Exception as exc: # noqa: BLE001 — surface verbatim - return [CheckResult( - checkpoint_id=cid, category=cat, priority=Priority.HIGH.value, + except Exception as exc: # noqa: BLE001 — API errors must be visible + return None, CheckResult( + checkpoint_id=checkpoint_id, + category=category, + priority=Priority.HIGH.value, + status=Status.WARNING.value, + description=description, + result=( + "Unable to read Declarative Agent components from " + f"AgentBuilder: {type(exc).__name__}: {exc}" + ), + remediation=( + "Retry after verifying AgentBuilder access. If this persists, " + f"report the checkpoint ID ({checkpoint_id}) and the error " + "above." + ), + doc_link=doc_link, + roles=roles, + ) + if schema_names is None: + return None, CheckResult( + checkpoint_id=checkpoint_id, + category=category, + priority=Priority.HIGH.value, status=Status.SKIPPED.value, - description="Workday write-scenario reference-data availability", - result=f"Unable to read Dataverse topic configuration: {exc}.", - remediation="Retry with Dataverse access, or review the topics manually in the maker portal.", + description=description, + result=( + "AgentBuilder client or active-agent botId not available — " + "cannot read botComponentChanges." + ), + remediation=( + "Run /setup so .local/config.json records the active agent " + "botId, then re-run FlightCheck with AgentBuilder access." + ), + doc_link=doc_link, roles=roles, - )] + ) + return schema_names, None - supported: set[str] = set() - getref_installed = False - requested_by_topic: dict[str, set[str]] = {} - for t in topics or []: - schema = t.get("schemaname") or "" - data = t.get("data") or "" - if "GetReferenceData" in schema: - getref_installed = True - supported |= _extract_supported_reference_keys(data) - continue - keys = _extract_requested_reference_keys(data) - if keys: - requested_by_topic[t.get("name") or schema or "(unnamed topic)"] = keys - if not requested_by_topic: - return [CheckResult( - checkpoint_id=cid, category=cat, priority=Priority.MEDIUM.value, - status=Status.NOT_CONFIGURED.value, - description="Workday write-scenario reference-data availability", - result="No Workday topic requests a reference-data picklist — nothing to validate.", - remediation="", - doc_link=doc, roles=roles, - )] +def _workday_topic_schema_names(schema_names: set[str]) -> list[str]: + return sorted(name for name in schema_names if ".topic.Workday" in name) - all_requested = sorted({k for ks in requested_by_topic.values() for k in ks}) - studio = _wd_studio_link(runner) - if not getref_installed: - return [CheckResult( - checkpoint_id=cid, category=cat, priority=Priority.HIGH.value, - status=Status.FAILED.value, - description="Workday write-scenario reference-data availability", - result=( - f"{len(requested_by_topic)} Workday topic(s) request reference-data picklists " - f"({', '.join(_ref_key_label(k) for k in all_requested)}), but the shared " - f"'GetReferenceData' topic is not installed — none of these picklists can be " - f"populated, so the agent cannot validate user input for these fields." - ), - remediation=( - "Install/repair the Workday extension so the 'GetReferenceData' system topic and " - "the msdyn_HRWorkdayHCMEmployeeGetReferenceData read scenario are present (they load " - "the reference picklists at runtime via a Workday report). " - f"Open {studio} \u2192 Topics to review the agent's Workday topics. See the Workday " - "topics + report-template configuration docs." - ), - doc_link=doc, roles=roles, - )] +def _workday_lookup_table_schema_names(schema_names: set[str]) -> list[str]: + return sorted( + name + for name in schema_names + if ".variable." in name and _schema_leaf(name).endswith("LookupTable") + ) + - gaps: dict[str, set[str]] = {} - for name, keys in requested_by_topic.items(): - missing = keys - supported - if missing: - gaps[name] = missing +def _check_workday_reference_data(runner) -> list[CheckResult]: + """WD-REF-001 — verify Workday reference-data components exist in DA.""" + roles = [Role.ESS_MAKER.value, Role.WORKDAY_ADMIN.value] + cid = "WD-REF-001" + cat = "Workday" + doc = f"{DOC_BASE}/workday" + description = "Workday write-topic reference-data components" + schema_names, terminal = _da_component_names_or_result( + runner, + checkpoint_id=cid, + category=cat, + description=description, + doc_link=doc, + roles=roles, + ) + if terminal is not None: + return [terminal] + assert schema_names is not None + + lookup_tables = _workday_lookup_table_schema_names(schema_names) + ref_topics = sorted( + name + for name in schema_names + if _schema_leaf(name) in _WD_REF_SYSTEM_TOPICS + ) + missing_ref_topics = sorted( + _WD_REF_SYSTEM_TOPICS - {_schema_leaf(name) for name in ref_topics} + ) + studio = _wd_studio_link(runner) - if not gaps: + if lookup_tables and not missing_ref_topics: return [CheckResult( checkpoint_id=cid, category=cat, priority=Priority.HIGH.value, status=Status.PASSED.value, - description="Workday write-scenario reference-data availability", + description=description, result=( - f"All {len(requested_by_topic)} Workday topic(s) that request reference-data " - f"picklists request only keys GetReferenceData supports " - f"({len(supported)} reference key(s) supported)." + f"botComponentChanges contains {len(lookup_tables)} Workday " + "LookupTable variable component(s) and both reference-data " + "topic component(s): " + + ", ".join(_schema_leaf(n) for n in lookup_tables + ref_topics) + + ". Structural enumeration only; reference-data key rules " + "remain gated pending the external catalog (US 7792327)." ), remediation="", - doc_link=doc, roles=roles, + doc_link=doc, + roles=roles, )] - lines = [ - f"'{name}': requests unsupported reference set(s) " - + ", ".join(f"{_ref_key_label(k)} [{k}]" for k in sorted(missing)) - for name, missing in sorted(gaps.items()) - ] + gaps = [] + if not lookup_tables: + gaps.append("0 .variable.*LookupTable component schemaName(s)") + if missing_ref_topics: + gaps.append( + "missing reference-data topic component schemaName(s): " + + ", ".join(missing_ref_topics) + ) return [CheckResult( checkpoint_id=cid, category=cat, priority=Priority.HIGH.value, status=Status.FAILED.value, - description="Workday write-scenario reference-data availability", + description=description, result=( - f"{len(gaps)} of {len(requested_by_topic)} Workday topic(s) request a reference-data " - f"picklist that GetReferenceData does not support, so that picklist cannot populate — " - f"the agent will reject valid inputs, accept invalid ones (downstream SOAP fault), or " - f"hallucinate allowed values:\n" + "\n".join(lines) + "botComponentChanges does not contain the Workday reference-data " + "components expected for Declarative Agent agents: " + + "; ".join(gaps) + + "." ), remediation=( - f"Open {studio} \u2192 Topics to fix the topic(s) above: either point each at a " - "reference key GetReferenceData supports, OR add the missing key to the " - "'GetReferenceData' topic and bind it to the Workday report that returns those " - "reference IDs (e.g. Get_Reference_IDs / Get_Countries). Until then the field has no " - "validated allowed-value list. See the Workday report-template + prompts-support " - "configuration docs." + f"Open {studio} and repair or re-import the Workday Declarative " + "Agent components so the LookupTable variables and " + "WorkdaySystemGetReferenceData / " + "WorkdaySystemRefreshReferenceData topics are present." ), - doc_link=doc, roles=roles, + doc_link=doc, + roles=roles, )] +def _should_run_da_component_inventory(runner) -> bool: + """Whether the Workday category runner should emit DA inventory rows.""" + if getattr(runner, "agentbuilder", None) is not None: + return True + scope = str(getattr(runner, "scope", "") or "") + if not scope.startswith("checkpoint:"): + return False + target = scope.removeprefix("checkpoint:").rstrip("*") + return target in {"WD-REF-001", "WD-WF-CAT-001"} + + def run_workday_checks(runner) -> list[CheckResult]: """Execute Workday-specific deep validation. @@ -644,6 +637,13 @@ def run_workday_checks(runner) -> list[CheckResult]: # when the kit-side Workday install isn't deployed yet. results.extend(_check_entra_workday_federation_alignment(runner)) + # DA structural inventory checks are flowless. They read the active + # agent's botComponentChanges directly, so run them before the legacy + # Workday flow gate below. + if _should_run_da_component_inventory(runner): + results.extend(_check_workday_reference_data(runner)) + results.extend(_check_custom_workflow_inventory(runner)) + # If neither flows nor any Workday connection references are # present, this tenant has no Workday integration. Skip the # downstream Workday-specific checks (preserves the pre-existing @@ -690,12 +690,6 @@ def run_workday_checks(runner) -> list[CheckResult]: # --- SOAP Workflow Tests (only if Workday MCP creds available) --- results.extend(_check_workflows(runner)) - # WD-REF-001 — write-scenario reference-data availability. Reconciles the - # reference picklists each installed Workday write scenario consumes against - # the shared GetReferenceData topic's supported keys (Dataverse-only; no - # external API). Config-level, so it runs regardless of SOAP credentials. - results.extend(_check_workday_reference_data(runner)) - # WD-SEC-003 — Personal Data domain write-permission probe. # Runs right after _check_workflows so it can reuse the same # ISU credentials the operator just supplied (no second prompt) @@ -707,16 +701,6 @@ def run_workday_checks(runner) -> list[CheckResult]: # connection checks above. results.extend(_check_package_connection_completeness(runner)) - # WD-WF-CAT-001 — Workday custom-workflow inventory checklist. - # Runs after the SOAP tests so the WD-WF-CAT-LINK trailer that - # `_check_workflows` emits inside its own returns has already - # populated `runner._workday_unknown_scenarios` (the trailer - # triggers the lazy discovery walk via _get_unknown_workday_scenarios). - # Emitting WD-WF-CAT-001 here ensures the full manual checklist - # appears in the per-Workday-block output even if there were no - # SOAP tests run (e.g. credentials unavailable). - results.extend(_check_custom_workflow_inventory(runner)) - return _suppress_manual_conn_sec_when_runs_healthy(results, runner) @@ -4268,10 +4252,8 @@ def _append_wd_wf_cat_link_trailer(runner, results: list[CheckResult]) -> None: "Workday Workflows block." ), remediation=( - "See WD-WF-CAT-001 for the per-scenario list and the " - "manual verification checklist (ISU account, payload " - "shape vs. Workday WSDL, evaluation test prompt, " - "connection-ref auth health)." + "See WD-WF-CAT-001 for the structural Workday topic " + "inventory read from the active agent's botComponentChanges." ), )) @@ -5153,116 +5135,13 @@ def _get_unknown_workday_scenarios(runner) -> list[dict]: return unknown -def _format_unknown_scenarios(unknown: list[dict]) -> str: - """Format the unknown-refs list for the result field. One block per - reference, agent + topic + line cited verbatim per AGENTS.md - principle #8 (result = what the kit observed). - """ - lines: list[str] = [] - for ref in unknown: - if ref["pattern"] == "system-common-execution": - name = ref.get("scenarioName") or "(scenarioName not found in topic)" - lines.append(f" • {name}") - lines.append( - f" Topic: topics/{ref['topic']}:{ref['line']}" - ) - lines.append(" Pattern: WorkdaySystemGetCommonExecution + scenarioName") - else: # invoke-flow-action - lines.append(f" • ({ref.get('flowId', '')})") - lines.append( - f" Topic: topics/{ref['topic']}:{ref['line']}" - ) - lines.append(" Pattern: InvokeFlowAction → flow bound to shared_workdaysoap") - lines.append(f" Agent: {ref['agent']}") - lines.append("") - return "\n".join(lines).rstrip() - - -_WD_WF_CAT_CHECKLIST = ( - "Manual verification required — the kit cannot validate custom " - "Workday scenarios end-to-end. For EACH scenario listed above:\n" - "\n" - " 1. ISU account: Confirm which ISU registered in Workday is used " - "by this scenario. Default is the account in environment variable " - "EmployeeContextRequestAccountName (see WD-ENV-001 output). Custom " - "scenarios may use a different ISU — verify in the template config " - "XML in Dataverse (Power Platform Maker → Tables → " - "msdyn_employeeselfservicetemplateconfigs → search ScenarioName).\n" - "\n" - " 2. Payload shape: Open the template config XML and confirm the " - "SOAP request body matches the Workday WSDL for the named service. " - "Field names, reference types, and required vs. optional elements " - "MUST match the Workday contract. Mismatches surface at runtime as " - "the 'Workflow Contract/Payload Mismatch' failure mode.\n" - "\n" - " 3. Test prompt: Add at least one evaluation test case to the " - "agent's evaluations/ folder that exercises the scenario end-to-end " - "with a known-good employee. Use /create-eval and tag the test set " - "with the scenario name.\n" - "\n" - " 4. Auth health: Re-check the WD-CONN-* connection token health " - "output for the connection reference this scenario uses. " - "Intermittent auth failures usually trace to a stale OAuth token " - "on one of the ISU refs — reauthenticate in Power Platform Maker " - "→ Connections.\n" - "\n" - "Note: the OOTB Workday catalog is resolved live from the " - "customer's own Dataverse " - "(msdyn_employeeselfservicetemplateconfigs, filtered by " - "ismanaged=true), which auto-detects every scenario the installed " - "Workday extension pack ships. A scenario surfacing as MANUAL " - "means it is NOT a managed row in the customer's tenant — either " - "it is genuinely custom (work through the checklist above) or the " - "extension pack is not installed.\n" - "\n" - "Found a new pattern? Log it back to the gap-discovery process. " - "File an issue at " - "https://github.com/microsoft/Employee-Self-Service-Agent-Developer-Kit/issues/new " - "with title 'Workday gap-discovery: ' if any of " - "the following apply:\n" - " • This MANUAL row surfaced a scenario that you believe should " - "ship OOTB in the Workday extension pack (forward to Microsoft " - "ESS so the next pack revision can include it).\n" - " • A Workday-bound topic in your agent did NOT surface here but " - "should have (the detection walker missed a new wiring pattern — " - "Pattern C or beyond; attach the topic YAML snippet so a new " - "detection rule can be added to _scan_topic_for_workday_refs).\n" - " • The 4-item checklist above was insufficient for diagnosing " - "your scenario (propose the additional verification step).\n" - "Include the topic YAML snippet, the scenarioName / flowId, and " - "the ADO incident number if any. Closing the loop here is how " - "WD-WF-CAT-001 gets better over time." -) - - def _check_custom_workflow_inventory(runner) -> list[CheckResult]: - """WD-WF-CAT-001 — Manual checklist for custom Workday workflows. - - Gates (in order): - * `runner._workday_package_flavor == "simplified"` → SKIPPED - (ISU/scenario inventory doesn't apply on the simplified - install per AGENTS.md principle #11). - * Missing `workspace/agents/` → SKIPPED. - * Zero Workday references discovered in any topic → SKIPPED. - * No Dataverse token / env URL → SKIPPED (we cannot resolve the - live OOTB catalog and must not return PASS per principle #1). - * Dataverse query errored → WARNING (surface the error per - principle #3 rather than silently swallowing it). - - Otherwise: - * Every discovered scenario is a managed row in Dataverse → PASSED. - * Any unknown / flow-bound reference → MANUAL with bucketed - listing in `result` and the 4-item checklist in `remediation`. - - Caches discovered list on `runner._workday_discovered_scenarios` - and the (possibly empty) unknown list on - `runner._workday_unknown_scenarios` so the WD-WF-CAT-LINK trailer - inside _check_workflows can read them without re-walking. - """ + """WD-WF-CAT-001 — inventory Workday DA topic components.""" cp_id = "WD-WF-CAT-001" category = "Workday Workflows" - description = "Workday custom-workflow inventory checklist" + description = "Workday topic component inventory" doc_link = f"{DOC_BASE}/workday-extensibility" + roles = [Role.ESS_MAKER.value] flavor = getattr(runner, "_workday_package_flavor", None) if flavor == "simplified": @@ -5272,137 +5151,47 @@ def _check_custom_workflow_inventory(runner) -> list[CheckResult]: category=category, )] - workspace_root = Path("workspace/agents") - if not workspace_root.exists(): - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id=cp_id, category=category, - priority=Priority.HIGH.value, status=Status.SKIPPED.value, - description=description, - result="workspace/agents/ directory not found.", - remediation=( - "Run /setup to extract agent files before this check " - "can enumerate Workday scenario references in topics." - ), - doc_link=doc_link, - )] - - # Discover Workday refs from topics first — no catalog needed for - # this step. If there are none, we can SKIP cleanly without even - # touching Dataverse. - discovered = _discover_customer_workday_scenarios(workspace_root) - runner._workday_discovered_scenarios = discovered - - if not discovered: - runner._workday_unknown_scenarios = [] - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id=cp_id, category=category, - priority=Priority.HIGH.value, status=Status.SKIPPED.value, - description=description, - result=( - "No Workday scenario references found in any agent topic. " - "Either Workday is not wired into the customer's agent yet, " - "or its topics are not yet extracted to " - "workspace/agents/*/topics/." - ), - remediation=( - "If the customer intends to use Workday, run /create to add " - "a Workday scenario topic, or /setup to re-extract topics " - "if you expected references to be present." - ), - doc_link=doc_link, - )] - - # We have Workday refs in topics — we need the live Dataverse - # OOTB catalog to know which are custom. No fallback: if Dataverse - # is unreachable, we cannot make a PASS/FAIL claim (principle #1). - catalog, status_code = _get_workday_ootb_catalog(runner) - - if catalog is None and status_code == "no_token": - # Suppress the trailer too — without the catalog we cannot - # legitimately list "unknowns." - runner._workday_unknown_scenarios = [] - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id=cp_id, category=category, - priority=Priority.HIGH.value, status=Status.SKIPPED.value, - description=description, - result=( - f"Found {len(discovered)} Workday scenario reference(s) " - "in customer topics, but no Dataverse credentials are " - "available to resolve the live OOTB scenario catalog " - "(msdyn_employeeselfservicetemplateconfigs where " - "ismanaged=true). Cannot determine which references are " - "custom vs. OOTB without that lookup." - ), - remediation=( - "Re-run flightcheck after running /setup so Dataverse " - "credentials are cached on the runner, or run it in an " - "environment where the Dataverse MCP server is " - "authenticated." - ), - doc_link=doc_link, - )] - - if catalog is None: - # query_error: — surface verbatim per principle #3. - err_msg = status_code.removeprefix("query_error: ") or "unknown error" - runner._workday_unknown_scenarios = [] - return [CheckResult(roles=[Role.ESS_MAKER.value], - checkpoint_id=cp_id, category=category, - priority=Priority.HIGH.value, status=Status.WARNING.value, - description=description, - result=( - f"Found {len(discovered)} Workday scenario reference(s) " - "in customer topics, but the Dataverse query for the " - "live OOTB scenario catalog " - "(msdyn_employeeselfservicetemplateconfigs where " - f"ismanaged=true) failed: {err_msg}. Cannot determine " - "which references are custom vs. OOTB until the query " - "succeeds." - ), - remediation=( - "Investigate the Dataverse error above. Common causes: " - "expired Dataverse token (re-run /setup), table not " - "present in this environment (ESS solution not " - "installed), or transient service outage (retry)." - ), - doc_link=doc_link, - )] - - # Catalog resolved cleanly — compute unknowns and cache them. - unknown = [ - ref for ref in discovered - if ref["pattern"] == "invoke-flow-action" - or (ref.get("scenarioName") and ref["scenarioName"] not in catalog) - ] - runner._workday_unknown_scenarios = unknown - - if not unknown: - return [CheckResult(roles=[Role.ESS_MAKER.value], + schema_names, terminal = _da_component_names_or_result( + runner, + checkpoint_id=cp_id, + category=category, + description=description, + doc_link=doc_link, + roles=roles, + ) + if terminal is not None: + return [terminal] + assert schema_names is not None + + workday_topics = _workday_topic_schema_names(schema_names) + if workday_topics: + topic_names = ", ".join(_schema_leaf(name) for name in workday_topics) + return [CheckResult(roles=roles, checkpoint_id=cp_id, category=category, priority=Priority.HIGH.value, status=Status.PASSED.value, description=description, result=( - f"Found {len(discovered)} Workday scenario reference(s) " - "in customer topics. All are managed rows in the " - "customer's Dataverse " - "(msdyn_employeeselfservicetemplateconfigs where " - "ismanaged=true) and require no manual review." + f"botComponentChanges contains {len(workday_topics)} " + f"Workday topic component(s): {topic_names}. Structural " + "enumeration only; shipped-vs-custom classification is gated " + "pending the external catalog (US 7792327)." ), + remediation="", doc_link=doc_link, )] - body = _format_unknown_scenarios(unknown) - return [CheckResult(roles=[Role.ESS_MAKER.value], + return [CheckResult(roles=roles, checkpoint_id=cp_id, category=category, - priority=Priority.HIGH.value, status=Status.MANUAL.value, + priority=Priority.HIGH.value, status=Status.FAILED.value, description=description, result=( - f"Found {len(unknown)} Workday scenario reference(s) in " - "customer topics that the kit cannot validate end-to-end " - "(not in the OOTB catalog):\n\n" - f"{body}" + "botComponentChanges contains 0 schemaName values matching the " + "Workday topic pattern (.topic.Workday*)." + ), + remediation=( + "Install or repair the Workday Declarative Agent extension so the " + "agent contains Workday topic components, then re-run FlightCheck." ), - remediation=_WD_WF_CAT_CHECKLIST, doc_link=doc_link, )] diff --git a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py index 9f023bc01..8ac4a47b1 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/checks/workday_extension.py @@ -20,11 +20,12 @@ (never assert a verdict from an unconfirmed API response shape) — this checkpoint echoes the observed ``connectionParametersSet.name`` for the operator to confirm, rather than PASS/FAIL on a guessed value. - * ``DV-CONN-001`` (S5.4) — the Dataverse connection reference the extension - pack ships (``…_92b66``, connector ``shared_commondataserviceforapps``) is - bound to an **active** connection, and its owner is echoed so the operator - can confirm it is their **own** account. Programmatic PASS/FAIL on a - documented-tier Dataverse ``connectionreferences`` read. + * ``DV-CONN-001`` (S5.4) — the Workday SOAP connection reference reported by + the Declarative Agent minimalBots components API is bound (``connectionId`` + present), and its owner is echoed so the operator can confirm it is their + **own** account. Programmatic PASS/FAIL on the validated minimalBots + components read (same endpoint + ``connectionReferenceChanges`` shape as the + shipped native ``DA-CONN-001`` check). * ``WD-REST-001`` (S5.5) — the captured ``restBaseUrl`` is present and **trimmed to** ``/api``. Pure-config check, no client. * ``WD-REST-002`` (S5.7) — the agent's ``user-context-setup.mcs.yml`` topic @@ -43,7 +44,7 @@ whole run. * **One CheckResult per checkpoint** (principle 7). * **No guessed API shapes** — the two API-backed checks read documented fields - only (Dataverse ``connectionid`` / ``statuscode``; BAP + only (minimalBots ``connectionReferenceChanges`` connector/connection ids; BAP ``connectionParametersSet.name`` / ``createdBy``), and degrade gracefully when a client is unavailable. * **Every** ``CheckResult`` declares ``roles=`` (enforced by @@ -52,18 +53,11 @@ from __future__ import annotations -import os import re -import sys from pathlib import Path from ..runner import CheckResult, Priority, Role, Status -# scripts/auth.py is on sys.path via cli.py at runtime (tests add it too); this -# mirrors checks/environment.py's top-level import so query_all is patchable as -# flightcheck.checks.workday_extension.query_all. -from auth import query_all # noqa: E402 - DOC_BASE = ( "https://learn.microsoft.com/en-us/copilot/microsoft-365/" "employee-self-service" @@ -91,12 +85,9 @@ _WORKDAY_RUNTIME_REF_LOGICAL_NAME = ( "msdyn_sharedworkdaysoap_workdayruntime" ) -# The Dataverse connection reference the simplified pack ships. -_DATAVERSE_CONNECTOR_SUFFIX = "/apis/shared_commondataserviceforapps" -_DATAVERSE_REF_SUFFIX = "92b66" -_DATAVERSE_RUNTIME_REF_LOGICAL_NAME = ( - "msdyn_sharedcommondataserviceforapps_workdayruntime" -) +# The Workday SOAP connection reference the Declarative Agent reports via the +# minimalBots components API (connector ``shared_workdaysoap``). +_WORKDAY_CONNECTOR_SUFFIX = "/apis/shared_workdaysoap" _REF_SUFFIX_RE = re.compile(r"_([0-9a-f]{5})$") # ---- Local user-context topic (WD-REST-002) ---- @@ -108,7 +99,7 @@ "Workday connection authentication type is Microsoft Entra ID Integrated" ) _DV_CONN_DESC = ( - "Dataverse connection reference bound to an active connection you own" + "Workday SOAP connection reference bound to a connection you own" ) _REST_URL_DESC = "Workday REST base URL present and trimmed to '/api'" _REDIRECT_DESC = ( @@ -155,14 +146,6 @@ def _is_workday_auth_ref(logical_name) -> bool: ) -def _is_dataverse_runtime_ref(logical_name) -> bool: - normalized = str(logical_name or "").casefold() - return ( - _ref_suffix(logical_name) == _DATAVERSE_REF_SUFFIX - or normalized == _DATAVERSE_RUNTIME_REF_LOGICAL_NAME.casefold() - ) - - def _host_of(url: str) -> str: """Return the host portion of an ``https://host/…`` URL for display.""" match = re.match(r"https?://([^/]+)", str(url).strip()) @@ -192,26 +175,48 @@ def _resolve_owner(props: dict) -> str: def _query_connection_references(runner): - """Return all Dataverse ``connectionreferences`` rows, or ``None`` when the - Dataverse token/endpoint is not available. - - Documented-tier read (Dataverse Web API v9.2) — no cassette required; tests - stub ``query_all``. + """Return the agent's connection references from the Declarative Agent + minimalBots components API, normalized to the row shape + ``_check_dv_connection`` consumes, or ``None`` when the AgentBuilder client + or the active-agent ``botId`` is unavailable. + + Validated-tier read (minimalBots ``POST …/components``). The same endpoint + and ``connectionReferenceChanges`` shape already back the shipped native + ``DA-CONN-001`` check (``checks/native_agent.py``); see + ``tests/fixtures/cassettes/INDEX.md`` and ``tests/mocks/ + agentbuilder_connectivity.py``. Fails loudly (lets the dispatcher degrade + this checkpoint to a WARNING) rather than overclaiming: an + ``AgentBuilderHTTPError`` propagates, and a 200 payload whose + ``connectionReferenceChanges`` is present but not a list raises + ``ValueError`` (mirrors ``native_agent._connection_references``). A missing + changeset is treated as "no references" (genuine absence), not an error. """ - env_url = getattr(runner, "env_url", None) - dv_token = getattr(runner, "dv_token", None) - if not env_url or not dv_token: + client = getattr(runner, "agentbuilder", None) + config = getattr(runner, "config", None) or {} + agent_id = (config.get("agent") or {}).get("botId") + if client is None or not agent_id: return None - # Belt-and-suspenders: keep scripts/ importable even if the module was - # imported before cli.py put it on the path. - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) - return query_all( - env_url, - dv_token, - "connectionreferences", - "connectionreferenceid,connectionreferencelogicalname," - "connectionreferencedisplayname,connectorid,connectionid,statuscode", - ) + changeset = client.fetch_components(agent_id) or {} + changes = changeset.get("connectionReferenceChanges") + if changes is None: + return [] + if not isinstance(changes, list): + raise ValueError( + "Component fetch returned invalid connectionReferenceChanges." + ) + refs = [] + for change in changes: + ref = (change or {}).get("connectionReference") or {} + refs.append( + { + "connectionreferencelogicalname": ref.get( + "connectionReferenceLogicalName" + ), + "connectorid": ref.get("connectorId"), + "connectionid": ref.get("connectionId"), + } + ) + return refs def _get_connections(runner): @@ -366,7 +371,7 @@ def _check_connection_auth(runner) -> list[CheckResult]: # ───────────────────────────────────────────────────────────────────── -# DV-CONN-001 — Dataverse connection reference binding (S5.4, PASS/FAIL). +# DV-CONN-001 — Workday SOAP connection reference binding (S5.4, PASS/FAIL). # ───────────────────────────────────────────────────────────────────── @@ -378,67 +383,44 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.SKIPPED.value, description=_DV_CONN_DESC, result=( - "Dataverse token not available — skipping the Dataverse " - "connection-reference check." + "AgentBuilder client or active-agent botId not available — " + "skipping the Workday connection-reference check." ), )] - dv_refs = [ - r - for r in refs - if str(r.get("connectorid") or "").lower().endswith( - _DATAVERSE_CONNECTOR_SUFFIX - ) - and _is_dataverse_runtime_ref( - r.get("connectionreferencelogicalname") - ) - ] - if len(dv_refs) > 1: - names = ", ".join( - sorted( - str(ref.get("connectionreferencelogicalname") or "(unnamed)") - for ref in dv_refs - ) - ) - return [CheckResult(roles=_MAKER_ROLES, - checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.WARNING.value, - description=_DV_CONN_DESC, - result=( - "Multiple ESS Dataverse connection references match the " - f"runtime and legacy package fingerprints: {names}. " - "FlightCheck cannot determine which reference is active." - ), - remediation=( - "Remove obsolete Workday package references, then rerun " - "FlightCheck against the remaining Dataverse binding." - ), - doc_link=_DOC_SIMPLIFIED, - )] - dv_ref = dv_refs[0] if dv_refs else None + wd_ref = next( + ( + r + for r in refs + if str(r.get("connectorid") or "") + .lower() + .endswith(_WORKDAY_CONNECTOR_SUFFIX) + ), + None, + ) - if dv_ref is None: + if wd_ref is None: return [CheckResult(roles=_MAKER_ROLES, checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.NOT_CONFIGURED.value, + priority=Priority.HIGH.value, status=Status.FAILED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"(\u2026_{_DATAVERSE_REF_SUFFIX}, connector " - "shared_commondataserviceforapps) was not found in this " - "environment." + "The ESS Workday SOAP connection reference (connector " + "shared_workdaysoap) was not found in the Declarative Agent " + "components payload." ), remediation=( - "Install/repair the Workday extension pack so its Dataverse " - "connection reference is created, then bind it to a Dataverse " - "connection you own." + "Install or repair the Workday extension pack so its Workday " + "SOAP connection reference is created, then bind it to a " + "Workday connection you own." ), doc_link=_DOC_SIMPLIFIED, )] - dv_ref_name = str(dv_ref.get("connectionreferencelogicalname")) - connection_id = dv_ref.get("connectionid") - statuscode = dv_ref.get("statuscode") + wd_ref_name = str( + wd_ref.get("connectionreferencelogicalname") or "(unnamed)" + ) + connection_id = wd_ref.get("connectionid") if not connection_id: return [CheckResult(roles=_MAKER_ROLES, @@ -446,31 +428,13 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.FAILED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is unbound " - "(connectionid=null)." - ), - remediation=( - "In Power Platform / Copilot Studio, bind the Dataverse " - "connection reference to an active Dataverse connection owned " - "by your own account." - ), - doc_link=_DOC_SIMPLIFIED, - )] - - if statuscode != 1: - return [CheckResult(roles=_MAKER_ROLES, - checkpoint_id="DV-CONN-001", category=_CATEGORY, - priority=Priority.HIGH.value, status=Status.FAILED.value, - description=_DV_CONN_DESC, - result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is bound but inactive " - f"(statuscode={statuscode})." + "The ESS Workday SOAP connection reference " + f"({wd_ref_name}) is unbound (connectionId=null)." ), remediation=( - "Re-authenticate or re-bind the Dataverse connection so its " - "status is active, using an account you own." + "In Power Platform / Copilot Studio, bind the Workday SOAP " + "connection reference to an active Workday connection owned by " + "your own account." ), doc_link=_DOC_SIMPLIFIED, )] @@ -490,9 +454,8 @@ def _check_dv_connection(runner) -> list[CheckResult]: priority=Priority.HIGH.value, status=Status.PASSED.value, description=_DV_CONN_DESC, result=( - "The ESS Dataverse connection reference " - f"({dv_ref_name}) is bound to an active " - "connection." + owner_note + "The ESS Workday SOAP connection reference " + f"({wd_ref_name}) is bound to a connection." + owner_note ), doc_link=_DOC_SIMPLIFIED, )] diff --git a/solutions/ess-maker-skills/scripts/flightcheck/registry.py b/solutions/ess-maker-skills/scripts/flightcheck/registry.py index 6040d49ff..d9c3ab47b 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/registry.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/registry.py @@ -378,6 +378,30 @@ class ResolvedPlan: roles=(Role.POWER_PLATFORM_ADMIN.value,), is_family=True, ), + # WD-REF-001 — Workday write-topic reference-data component inventory. + # Reads DA botComponentChanges via AgentBuilder; no Dataverse endpoint. + CheckpointSpec( + key="WD-REF-001", + category_fn=run_workday_checks, + category_label="Workday", + clients=frozenset({AGENTBUILDER}), + requires_config=True, + requires_dataverse_endpoint=False, + priority=Priority.HIGH.value, + roles=(Role.ESS_MAKER.value, Role.WORKDAY_ADMIN.value), + ), + # WD-WF-CAT-001 — Workday topic component inventory. Exact entry must + # beat the legacy WD-WF family so the DA structural check stays flowless. + CheckpointSpec( + key="WD-WF-CAT-001", + category_fn=run_workday_checks, + category_label="Workday", + clients=frozenset({AGENTBUILDER}), + requires_config=True, + requires_dataverse_endpoint=False, + priority=Priority.HIGH.value, + roles=(Role.ESS_MAKER.value,), + ), # WD-WF-* — per-workflow SOAP runtime checks (skipped on the simplified # flavor; registered for completeness). Emitted with category # "Workday Workflows" but owned by run_workday_checks. @@ -521,15 +545,16 @@ class ResolvedPlan: priority=Priority.HIGH.value, roles=(Role.ESS_MAKER.value,), ), - # DV-CONN-001 — self-contained Dataverse read (its own connectionreferences - # query) plus a best-effort BAP owner echo. + # DV-CONN-001 — reads the Workday SOAP connection reference from the + # Declarative Agent minimalBots components API (AGENTBUILDER), plus a + # best-effort BAP owner echo (PP_ADMIN). CheckpointSpec( key="DV-CONN-001", category_fn=run_workday_extension_checks, category_label="Workday Extension", - clients=frozenset({DATAVERSE, PP_ADMIN}), + clients=frozenset({AGENTBUILDER, PP_ADMIN}), requires_config=True, - requires_dataverse_endpoint=True, + requires_dataverse_endpoint=False, priority=Priority.HIGH.value, roles=(Role.ESS_MAKER.value,), ), @@ -635,6 +660,7 @@ class ResolvedPlan: "WD-CONN", "WD-RUN", "WD-FLOW", + "WD-REF", "WD-WF", "WD-ENV", "WD-ENTRA", diff --git a/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md b/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md index a973217b9..e0fc9d0af 100644 --- a/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md +++ b/solutions/ess-maker-skills/src/reference/ess-docs/flightcheck/validation-matrix.md @@ -249,49 +249,31 @@ Tests all 17 ESS pre-configured workflows against the Workday API. Requires ISU | WD-WF-016 | Update Email | Human_Resources | Write | | Contact Information (see WD-SEC-003 for the precise Personal Data + Maintain Contact Information / Edit Worker Additional Data check) | | WD-WF-017 | Update Phone | Human_Resources | Write | | Contact Information (see WD-SEC-003 for the precise Personal Data + Maintain Contact Information / Edit Worker Additional Data check) | -### Workday Custom-Workflow Inventory (WD-WF-CAT-xxx) — Manual - -The 17 SOAP tests above cover only the OOTB workflows the kit ships -SOAP envelopes for. Customers routinely wire up additional Workday -scenarios via two patterns: - -- **Pattern A** — Topic that calls - `WorkdaySystemGetCommonExecution` with a `scenarioName` of a - template-config record in Dataverse. -- **Pattern B** — Standalone topic that calls a customer-built cloud - flow bound to the `shared_workdaysoap` connector via - `InvokeFlowAction`. - -Both patterns exit the automated validation surface (the kit doesn't -ship a Workday WSDL parser; per-tenant security domain / ISU config -varies). WD-WF-CAT-001 walks `workspace/agents/*/topics/*.mcs.yml` -for these patterns and emits a MANUAL row enumerating any scenarios -that aren't in the OOTB catalog. The OOTB catalog is resolved live -from the customer's own Dataverse: the check queries -`msdyn_employeeselfservicetemplateconfigs` and treats every -`ismanaged=true` row as OOTB (auto-detects every scenario shipped by -the installed Workday extension pack, with no kit-side curation -needed). There is no fallback — if Dataverse credentials are missing -the check returns SKIPPED, and if the Dataverse query errors the -check returns WARNING surfacing the error verbatim. Returning PASSED -without a tenant-accurate catalog would violate FlightCheck design -principle #1 ("never return PASSED when the check cannot actually -validate what it claims to validate"). The MANUAL remediation -carries a 4-item checklist (ISU account, payload shape vs. Workday -WSDL, evaluation test prompt, connection-ref auth health). +### Workday DA Component Inventory (WD-REF-001, WD-WF-CAT-001) + +These checks read the active Declarative Agent's `botComponentChanges` +through `runner.agentbuilder.fetch_components` and validate structural +component `schemaName` values. They do not walk local topic YAML and +do not diff Dataverse managed template configs. + +Status mapping: required schemaNames present returns PASSED, required +schemaNames missing returns FAILED, missing AgentBuilder access or +active-agent bot ID returns SKIPPED, and malformed +`botComponentChanges` or AgentBuilder read errors return WARNING. +WD-WF-CAT-001 also returns SKIPPED before the API read on a simplified +Workday install. | ID | Check | Priority | Method | Doc Link | |----|-------|----------|--------|----------| -| WD-WF-CAT-001 | Workday custom-workflow inventory checklist (MANUAL) | High | Local file walk + Dataverse managed-template-config diff (SKIP on no token, WARN on query error) | [workday-extensibility](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/workday-extensibility) | +| WD-REF-001 | Workday reference-data component inventory | High | AgentBuilder `botComponentChanges`: requires `.variable.*LookupTable` variables plus `WorkdaySystemGetReferenceData` and `WorkdaySystemRefreshReferenceData` topic components | [workday](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/workday) | +| WD-WF-CAT-001 | Workday topic component inventory | High | AgentBuilder `botComponentChanges`: enumerates `.topic.Workday*` schemaNames as the structural Workday topic inventory | [workday-extensibility](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/workday-extensibility) | | WD-WF-CAT-LINK | Cross-link trailer surfacing WD-WF-CAT-001 from inside the SOAP-test block | Medium | Computed from WD-WF-CAT-001 cache | [workday-extensibility](https://learn.microsoft.com/en-us/copilot/microsoft-365/employee-self-service/workday-extensibility) | -MANUAL rows do not fail readiness (per FlightCheck design principle #2) -— they direct the operator to verify what the kit cannot. Address -each scenario by confirming it against the 4-item checklist in the -customer's environment. A scenario surfacing as MANUAL means it is -NOT a managed row in the customer's tenant — either it is genuinely -custom or the Workday extension pack is not installed in this -environment. +WD-WF-CAT-LINK is still a Medium MANUAL trailer emitted from the +SOAP-test block when customer topics reference Workday scenarios +outside the automated SOAP-test catalog. It points operators to the +WD-WF-CAT-001 structural topic inventory so they can see which +Workday topic components the active Declarative Agent currently has. ## 6. Local Agent File Validation (Kit-exclusive) diff --git a/tests/flightcheck/checks/test_workday_custom_inventory.py b/tests/flightcheck/checks/test_workday_custom_inventory.py index f5358183c..ec1f704ed 100644 --- a/tests/flightcheck/checks/test_workday_custom_inventory.py +++ b/tests/flightcheck/checks/test_workday_custom_inventory.py @@ -1,937 +1,131 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Tests for WD-WF-CAT-001 (Workday custom-workflow inventory checklist) -and its WD-WF-CAT-LINK cross-link trailer emitted from inside -`_check_workflows`. - -These are PURE-LOGIC tests: the production check makes NO external API -calls — it reads `workspace/agents/*/topics/*.mcs.yml` and -`workspace/agents/*/workflows/*/workflow.json` from the local -filesystem. Per `tests/AGENTS.md` "The cardinal rule does not apply -to: Tests of the kit's pure-logic helpers (no network)." — no -cassettes or mock-tier enforcement needed. - -The check covers the WD-001 acceptance criteria (ADO 7392277, -incidents 760098889 / 783902203): - - AC1: Checklist enumerates: which custom Workday workflows are wired - up, ISU account used, expected payload shape, test prompt. - AC2: Linked from Test-WorkdayWorkflows output when an unknown - workflow name is referenced in customer topics. - AC3: Includes a "found a new pattern? log it here" loop back to - the gap-discovery process. - -Tests below pin each AC explicitly so a future drive-by refactor that -weakens the checklist text fails CI rather than silently ships a -weaker remediation. -""" +"""Tests for WD-WF-CAT-001 DA Workday topic component inventory.""" from __future__ import annotations -import json from dataclasses import dataclass, field -from pathlib import Path from typing import Any -import pytest -import responses +from tests.conftest import require_validated_mock +from tests.mocks import agentbuilder_connectivity as ab -from tests.conftest import ( - FAKE_DATAVERSE_URL, - FAKE_TOKEN, - require_validated_mock, -) -from tests.mocks import dataverse as dv +require_validated_mock(ab) -require_validated_mock(dv) +from flightcheck.checks.workday import _check_custom_workflow_inventory # noqa: E402 +from flightcheck.runner import Status # noqa: E402 -# ───────────────────────────────────────────────────────────────────────── -# Minimal runner — _check_custom_workflow_inventory only reads -# `_workday_package_flavor` and writes the cached `_workday_*` lists. -# ───────────────────────────────────────────────────────────────────────── +class _FakeAgentBuilder: + def __init__(self, payload: dict[str, Any]): + self._payload = payload - -@dataclass -class _MinimalRunner: - config: dict[str, Any] = field(default_factory=dict) - - -def _result_by_id(results: list, checkpoint_id: str): - matches = [r for r in results if r.checkpoint_id == checkpoint_id] - assert len(matches) == 1, ( - f"Expected exactly one result for {checkpoint_id}, got {len(matches)}: " - f"{[r.checkpoint_id for r in results]}" - ) - return matches[0] - - -# ───────────────────────────────────────────────────────────────────────── -# Workspace fixtures — small helpers that lay down realistic -# `workspace/agents//topics/` + `workspace/agents//workflows/` -# trees under a tmp_path, matching the on-disk shape exactly so the -# production walkers don't need to be parameterized for testing. -# ───────────────────────────────────────────────────────────────────────── - - -def _write_topic_system_common_execution( - topic_path: Path, *, scenario_name: str -) -> None: - """Pattern A: scenarioName + WorkdaySystemGetCommonExecution dialog. - - Layout mirrors `src/examples/ess-samples/Workday/ManagerScenarios/ - WorkdayManagersdirect-CompanyCode/topic.yaml` lines 38-46. - """ - topic_path.parent.mkdir(parents=True, exist_ok=True) - topic_path.write_text( - "kind: AdaptiveDialog\n" - "beginDialog:\n" - " kind: OnRecognizedIntent\n" - " id: test-topic\n" - " actions:\n" - " - kind: BeginDialog\n" - " id: Gt044B\n" - " displayName: Redirect to Workday Get Common Execution\n" - " input:\n" - " binding:\n" - ' parameters: ="{\\"params\\":[]}"\n' - f" scenarioName: {scenario_name}\n" - "\n" - " dialog: msdyn_copilotforemployeeselfservicehr.topic.WorkdaySystemGetCommonExecution\n" - " output:\n" - " binding:\n" - " errorResponse: Topic.errorResponse\n", - encoding="utf-8", - ) - - -def _write_topic_invoke_flow_action( - topic_path: Path, *, flow_id: str -) -> None: - """Pattern B: kind: InvokeFlowAction with a flowId pointing at a - Workday-bound flow. Layout mirrors the canonical InvokeFlowAction - shape in `src/examples/ess-samples/Facilities/.../topic.yaml`.""" - topic_path.parent.mkdir(parents=True, exist_ok=True) - topic_path.write_text( - "kind: AdaptiveDialog\n" - "beginDialog:\n" - " kind: OnRecognizedIntent\n" - " actions:\n" - " - kind: InvokeFlowAction\n" - " id: invoke-1\n" - f" flowId: {flow_id}\n" - " input:\n" - " binding:\n" - " text: =Topic.UserQuery\n" - " output:\n" - " binding:\n" - " response: Topic.Response\n", - encoding="utf-8", - ) + def fetch_components(self, _agent_id: str) -> dict[str, Any]: + return self._payload -def _write_workflow( - agent_dir: Path, - *, - workflow_id: str, - workflow_slug: str, - api_name: str, -) -> None: - """Write a minimal workflow folder with `metadata.yml` + - `workflow.json`. `api_name` controls whether the flow is - Workday-bound — set to `shared_workdaysoap` to make this a Workday - flow, anything else (e.g. `shared_servicenow`) makes it - non-Workday.""" - wf_dir = agent_dir / "workflows" / workflow_slug - wf_dir.mkdir(parents=True, exist_ok=True) - (wf_dir / "metadata.yml").write_text( - f"workflowId: {workflow_id}\n" - "jsonFileName: workflow.json\n", - encoding="utf-8", - ) - (wf_dir / "workflow.json").write_text( - json.dumps({ - "properties": { - "connectionReferences": { - "primary": { - "api": {"name": api_name} - } - } - } - }), - encoding="utf-8", +@dataclass +class _Runner: + config: dict[str, Any] = field( + default_factory=lambda: {"agent": {"botId": ab.MOCK_AGENT_ID}} ) - - -def _write_catalog_marker_file(tmp_path: Path) -> None: - """Make sure `Path("workspace/agents")` resolves relative to - `tmp_path` (the test chdir'd here in the autouse fixture). The - discovery walker uses `Path("workspace/agents")` directly — a - relative path resolved against CWD — so no marker file is needed, - but a `workspace/` dir must exist or the walker returns [] (which - triggers the "directory not found" SKIP path rather than the - "no Workday refs" SKIP path).""" - (tmp_path / "workspace" / "agents").mkdir(parents=True, exist_ok=True) - - -# ───────────────────────────────────────────────────────────────────────── -# Class — keeps the autouse env-isolation fixture from leaking into -# unrelated test files. Mirrors the structure of -# `test_workday_workflows_gate.py::TestSimplifiedInstallGate`. -# ───────────────────────────────────────────────────────────────────────── - - -class TestCustomWorkflowInventory: - @pytest.fixture(autouse=True) - def _isolate_env( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path - ) -> None: - """The production check uses `Path("workspace/agents")` relative - to CWD. Tests must chdir into tmp_path so each test sees the - workspace it just built and is isolated from any sibling test - AND from the developer's repo (which has a real - `workspace/agents/` for `employee-self-service-it`).""" - monkeypatch.chdir(tmp_path) - - # ------------------------------------------------------------------ - # Gates - # ------------------------------------------------------------------ - - def test_simplified_install_skips(self) -> None: - """Principle #11: simplified install (no ISU) has no concept of - ISU/scenario inventory — the check must short-circuit via the - canonical `_simplified_install_skip` helper. Pin that the SKIP - carries WD-PKG-001 reasoning so the operator knows WHY this - check skipped (vs e.g. a credential-missing skip).""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - runner = _MinimalRunner() - runner._workday_package_flavor = "simplified" - - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Skipped" - assert r.category == "Workday Workflows" - # `_simplified_install_skip` produces a result that names - # WD-PKG-001 as the gating check — pin that contract so a - # future refactor that bypasses the shared helper fails. - assert "WD-PKG-001" in r.result - assert "simplified" in r.result.lower() - - def test_no_workspace_directory_skips(self) -> None: - """If `/setup` hasn't run, there's no `workspace/agents/` and the - discovery walker has nothing to scan. SKIP with a result that - names the missing directory verbatim and a remediation that - directs the operator to `/setup` (so this isn't mistaken for - a "no Workday in customer's agent" PASS).""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - runner = _MinimalRunner() - # No workspace/agents/ — _isolate_env chdir'd to a clean tmp_path. - - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Skipped" - assert "workspace/agents/" in r.result - assert "/setup" in r.remediation - - def test_workspace_exists_but_zero_refs_skips(self, tmp_path: Path) -> None: - """A workspace with topics but ZERO Workday references is not a - PASS — the customer may simply not use Workday. SKIP with a - message that names both possibilities (not wired up vs. not - extracted) so the operator can pick the right next step.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - _write_catalog_marker_file(tmp_path) - agent_dir = tmp_path / "workspace" / "agents" / "test-agent" - (agent_dir / "topics").mkdir(parents=True, exist_ok=True) - # Topic that mentions neither WorkdaySystemGetCommonExecution - # nor an InvokeFlowAction bound to shared_workdaysoap. - (agent_dir / "topics" / "greeting.mcs.yml").write_text( - "kind: AdaptiveDialog\n" - "beginDialog:\n" - " kind: OnRecognizedIntent\n" - " actions:\n" - " - kind: SendActivity\n" - " activity: Hello!\n", - encoding="utf-8", - ) - - runner = _MinimalRunner() - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Skipped" - # Pin both possibilities so the message stays operator-actionable. - assert "No Workday scenario references" in r.result - assert "not wired into" in r.result - assert "/create" in r.remediation or "/setup" in r.remediation - - # ------------------------------------------------------------------ - # Catalog matching — MANUAL path (the core acceptance criterion) - # ------------------------------------------------------------------ - - @responses.activate - def test_unknown_scenario_emits_manual_with_full_checklist( - self, tmp_path: Path - ) -> None: - """The primary acceptance criterion (AC1): an unknown scenario - surfaces as one MANUAL row with the scenario name in `result` - and the 4-item checklist (ISU / payload / test prompt / auth) - in `remediation`. Pins every checklist item literally so a - drive-by edit that drops one fails CI.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "custom.mcs.yml", - # A clearly-not-shipped name. If a managed row ever has this - # exact name the test breaks loudly — that's the intended - # behaviour. - scenario_name="msdyn_HRCustomNotInCatalogXYZ_TestOnly", - ) - # Empty managed-row response → every discovered scenario is - # treated as custom and surfaces as MANUAL. - _register_template_configs_response(rows=[]) - - runner = _RunnerWithDataverse() - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - - # Per AGENTS.md principle #2: MANUAL = "Manual" — must NOT be - # "Failed" or "Warning". MANUAL does not fail readiness. - assert r.status == "Manual", ( - f"Custom scenario must surface as MANUAL (per AGENTS.md " - f"principle #2), got {r.status!r}" - ) - assert r.priority == "High" - - # `result` (AGENTS.md principle #8: what the kit observed): - # must name the scenario verbatim + cite the topic file + line. - assert "msdyn_HRCustomNotInCatalogXYZ_TestOnly" in r.result - assert "topics/custom.mcs.yml" in r.result - assert "WorkdaySystemGetCommonExecution" in r.result - assert "ess-hr" in r.result # agent slug - - # `remediation` (AGENTS.md principle #8: action only). All four - # checklist items (AC1 verbatim from the ticket) must appear. - assert "ISU account" in r.remediation - assert "Payload shape" in r.remediation - assert "Test prompt" in r.remediation - assert "Auth health" in r.remediation - # Pin specific actionable phrases the operator needs to act: - assert "msdyn_employeeselfservicetemplateconfigs" in r.remediation - assert "/create-eval" in r.remediation - # AC3 ("found a new pattern? log it here" loop-back) is pinned - # separately by test_checklist_includes_gap_discovery_loopback - # below — keep that test in lockstep with the AC3 paragraph in - # _WD_WF_CAT_CHECKLIST. - - @responses.activate - def test_checklist_includes_gap_discovery_loopback( - self, tmp_path: Path - ) -> None: - """AC3: the MANUAL remediation MUST include a "found a new - pattern? log it here" paragraph that closes the loop back to - the kit's gap-discovery process. Without it, customers who hit - a detection gap (e.g. a Pattern C wiring the walker doesn't - catch) or a scenario they believe should ship OOTB have no - canonical channel to forward that signal, and WD-WF-CAT-001 - can't improve over time. - - The original commit eb02d32 included this loop-back text; the - Dataverse-API refactor (0fb2383) dropped it on the rationale - that kit-side PRs aren't the remediation for the catalog - anymore. But AC3 is broader than the catalog — it covers - detection-pattern gaps and OOTB-promotion feedback too. Pin - the restored paragraph here so a future drive-by edit that - re-strips it fails CI, not silently ships a weaker checklist. - """ - from flightcheck.checks.workday import _check_custom_workflow_inventory - - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "custom.mcs.yml", - scenario_name="msdyn_HRCustomAC3Test_Unknown", - ) - # Empty managed-row response → scenario surfaces as MANUAL so - # the full _WD_WF_CAT_CHECKLIST renders (the loop-back text - # is part of the same checklist, not a separate row). - _register_template_configs_response(rows=[]) - - runner = _RunnerWithDataverse() - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Manual" - - # The verbatim AC3 framing phrase from the ticket. If this - # disappears the next time someone refactors the checklist, - # CI must catch it. - assert "Found a new pattern" in r.remediation, ( - "AC3 loop-back framing dropped from checklist — operators " - "have no canonical channel to log gap-discovery feedback" - ) - assert "gap-discovery process" in r.remediation, ( - "AC3 must explicitly name the gap-discovery process so " - "operators understand where the loop closes" - ) - # The actionable channel: the kit repo's issues page. Pin the - # exact URL — a typo'd link is worse than no link (operator - # files an issue against a 404 and the signal is lost). - assert ( - "https://github.com/microsoft/" - "Employee-Self-Service-Agent-Developer-Kit/issues/new" - ) in r.remediation, ( - "AC3 loop-back must link to the kit repo issues page so " - "feedback reaches the team that owns WD-WF-CAT-001" - ) - # The three gap categories the loop-back exists to capture — - # if any are dropped, the loop closes on a narrower set of - # signals than AC3 requires. - assert "should ship OOTB" in r.remediation, ( - "AC3 must invite OOTB-promotion feedback (scenarios " - "customers routinely build custom that Microsoft should " - "ship in the extension pack)" - ) - assert "detection walker" in r.remediation, ( - "AC3 must invite detection-pattern feedback (a topic " - "wiring shape the walker missed)" - ) - assert "checklist above was insufficient" in r.remediation, ( - "AC3 must invite checklist-completeness feedback (the " - "4-item checklist itself can grow as new failure modes " - "are discovered)" - ) - - # ------------------------------------------------------------------ - # Pattern B (InvokeFlowAction → Workday-bound flow) - # ------------------------------------------------------------------ - - @responses.activate - def test_invoke_flow_action_workday_bound_emits_manual( - self, tmp_path: Path - ) -> None: - """Pattern B: a topic that calls a custom cloud flow bound to - `shared_workdaysoap` is ALWAYS unknown (Dataverse template - configs key by scenarioName; customer-built flow GUIDs don't - appear there) — surface MANUAL with the flow GUID + topic - location named verbatim.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - flow_id = "9f1b2c3d-aaaa-bbbb-cccc-111111111111" - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_invoke_flow_action( - agent_dir / "topics" / "custom-flow.mcs.yml", - flow_id=flow_id, - ) - _write_workflow( - agent_dir, - workflow_id=flow_id, - workflow_slug="ess-hr-workday-9f1b2c3d-xxxx", - api_name="shared_workdaysoap", - ) - # Dataverse returns some managed rows — but flow-bound refs - # are ALWAYS unknown regardless of catalog contents, so this - # test still gets MANUAL. - _register_template_configs_response(rows=[ - _template_config_row(name="msdyn_SomeOtherScenario", ismanaged=True), - ]) - - runner = _RunnerWithDataverse() - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Manual" - assert "topics/custom-flow.mcs.yml" in r.result - assert flow_id in r.result - assert "InvokeFlowAction" in r.result - assert "shared_workdaysoap" in r.result - # `flow-bound, no scenarioName` is the literal label - # _format_unknown_scenarios uses for Pattern B refs — pin it - # so an operator scanning the output can tell at a glance - # this is a flow ref, not a scenarioName ref. - assert "flow-bound" in r.result - - def test_invoke_flow_action_non_workday_is_ignored( - self, tmp_path: Path - ) -> None: - """Conservative qualification (Pattern B): a flow bound to e.g. - `shared_servicenow` must NOT surface in this check — only - Workday-connected flows do. Otherwise this check would emit - false positives for every customer's ServiceNow integration.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - flow_id = "abcdef01-2222-3333-4444-555555555555" - agent_dir = tmp_path / "workspace" / "agents" / "ess-it" - _write_topic_invoke_flow_action( - agent_dir / "topics" / "create-ticket.mcs.yml", - flow_id=flow_id, - ) - _write_workflow( - agent_dir, - workflow_id=flow_id, - workflow_slug="ess-it-servicenow-aaaa", - api_name="shared_servicenow", - ) - - runner = _MinimalRunner() - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - # Zero Workday refs found → SKIPPED with the "not wired in" - # message, NOT a MANUAL row mentioning the ServiceNow flow. - assert r.status == "Skipped" - assert "No Workday scenario references" in r.result - assert flow_id not in r.result, ( - "ServiceNow-bound flow leaked into Workday inventory output — " - "_is_workday_bound_workflow_json qualification regressed" - ) - - # ------------------------------------------------------------------ - # Bucketing (AGENTS.md principle #7) - # ------------------------------------------------------------------ - - @responses.activate - def test_multiple_unknowns_bucket_into_single_row( - self, tmp_path: Path - ) -> None: - """Principle #7: N unknown scenarios collapse to ONE MANUAL row - listing all N in `result`, with the de-duplicated 4-item - checklist as the SINGLE `remediation`. If a future refactor - emits one row per scenario, the operator sees the checklist - N times — exactly what bucketing exists to prevent.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - for n in range(3): - _write_topic_system_common_execution( - agent_dir / "topics" / f"custom-{n}.mcs.yml", - scenario_name=f"msdyn_HRCustomBucketTest_{n}", + agentbuilder: Any = None + _workday_package_flavor: str | None = None + + +def _runner_with_components(schema_names: list[str]) -> _Runner: + return _Runner( + agentbuilder=_FakeAgentBuilder( + ab.components_with_bot_components( + bot_components=[ + ab.bot_component_change(schema_name=name) + for name in schema_names + ] ) - # Empty managed-row response → all 3 surface as unknown. - _register_template_configs_response(rows=[]) - - runner = _RunnerWithDataverse() - results = _check_custom_workflow_inventory(runner) - - # Exactly one row, NOT three. - cat_rows = [r for r in results if r.checkpoint_id == "WD-WF-CAT-001"] - assert len(cat_rows) == 1, ( - f"Bucketing regressed: got {len(cat_rows)} WD-WF-CAT-001 rows, " - f"expected 1 (per AGENTS.md principle #7)" - ) - - r = cat_rows[0] - assert r.status == "Manual" - # All three scenario names appear in `result`. - for n in range(3): - assert f"msdyn_HRCustomBucketTest_{n}" in r.result, ( - f"Scenario {n} dropped from bucketed result" - ) - # The checklist appears ONCE in remediation, not three times. - assert r.remediation.count("ISU account") == 1, ( - "Checklist appears multiple times in remediation — bucketing " - "should emit a single de-duplicated checklist" - ) - - # ------------------------------------------------------------------ - # Caching contract (the check + the cross-link trailer share state) - # ------------------------------------------------------------------ - - @responses.activate - def test_discovery_results_cached_on_runner(self, tmp_path: Path) -> None: - """`_check_custom_workflow_inventory` caches discovery on the - runner so the topic walk runs at most once per flightcheck. - Without this, `_check_workflows` (trailer) + the main check - would walk every topic twice. Pin that both caches populate - on first read.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "custom.mcs.yml", - scenario_name="msdyn_HRCustomCacheTest", - ) - # Empty managed-row response so the scenario surfaces as - # unknown (which is what the cache needs to capture). - _register_template_configs_response(rows=[]) - - runner = _RunnerWithDataverse() - # Pre-condition: cache attributes absent. - assert not hasattr(runner, "_workday_unknown_scenarios") - assert not hasattr(runner, "_workday_discovered_scenarios") - - _check_custom_workflow_inventory(runner) - - # Post-condition: both caches populated. - assert hasattr(runner, "_workday_unknown_scenarios") - assert hasattr(runner, "_workday_discovered_scenarios") - assert len(runner._workday_unknown_scenarios) == 1 - assert ( - runner._workday_unknown_scenarios[0]["scenarioName"] - == "msdyn_HRCustomCacheTest" ) + ) -# ───────────────────────────────────────────────────────────────────────── -# Cross-link trailer (WD-WF-CAT-LINK) — emitted from inside -# `_check_workflows`, satisfies AC2 ("Linked from Test-WorkdayWorkflows -# output"). Lives in its own class because it exercises the -# simplified-install gate of `_check_workflows`, not -# `_check_custom_workflow_inventory`. -# ───────────────────────────────────────────────────────────────────────── - - -class TestCrossLinkTrailer: - @pytest.fixture(autouse=True) - def _isolate_env( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path - ) -> None: - # Mirror the isolation in test_workday_workflows_gate.py — when - # the simplified gate does NOT fire, `_check_workflows` reaches - # `_resolve_workday_metadata` which reads env vars + mcp.json. - # Strip ambient state so tests are deterministic. - monkeypatch.delenv("WORKDAY_BASE_URL", raising=False) - monkeypatch.delenv("WORKDAY_TENANT", raising=False) - monkeypatch.delenv("WORKDAY_TEST_EMPLOYEE_ID", raising=False) - monkeypatch.chdir(tmp_path) - - @responses.activate - def test_trailer_emitted_when_unknowns_exist(self, tmp_path: Path) -> None: - """AC2: when an unknown Workday workflow name is referenced in - customer topics, the SOAP-test output MUST emit a cross-link - row pointing at WD-WF-CAT-001 so an admin reading a clean - 17-row pass doesn't miss the manual row below it. - - Note: the trailer relies on `_get_unknown_workday_scenarios` - which in turn needs the live Dataverse catalog. Without - credentials the inventory check SKIPs and the trailer - self-suppresses (the main SKIPPED row is the single source of - truth). So this test wires up env_url/dv_token + a mocked - empty managed-row response so the unknown surfaces.""" - from flightcheck.checks.workday import _check_workflows - - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "custom.mcs.yml", - scenario_name="msdyn_HRCustomTrailerTest", - ) - # Empty managed-row response → topic scenario surfaces as - # unknown → trailer fires. - _register_template_configs_response(rows=[]) - - @dataclass - class R: - env_url: str = FAKE_DATAVERSE_URL - dv_token: str = FAKE_TOKEN - config: dict = field(default_factory=dict) - - runner = R() - # Don't set _workday_package_flavor at all → gate doesn't fire - # (falls through to credential-missing path which emits the - # WD-WF-000 SKIP row, then the trailer code runs). - - results = _check_workflows(runner) - - # The SKIP row for credentials missing must exist (sanity). - wd_wf_000 = [r for r in results if r.checkpoint_id == "WD-WF-000"] - assert len(wd_wf_000) == 1 - assert wd_wf_000[0].status == "Skipped" - - # AC2: the trailer row exists and references the main check. - trailer = [r for r in results if r.checkpoint_id == "WD-WF-CAT-LINK"] - assert len(trailer) == 1, ( - f"Trailer WD-WF-CAT-LINK missing — AC2 regression. " - f"Got rows: {[r.checkpoint_id for r in results]}" - ) - t = trailer[0] - assert t.status == "Manual" - assert t.category == "Workday Workflows" - assert "WD-WF-CAT-001" in t.remediation, ( - "Trailer must cross-link to WD-WF-CAT-001 — operators read " - "the trailer text to find the full checklist" - ) - assert "1 Workday scenario reference" in t.result, ( - f"Trailer must name the count + the bucket — got {t.result!r}" - ) - - def test_trailer_absent_when_clean(self, tmp_path: Path) -> None: - """When there are zero unknown Workday refs, the trailer MUST - NOT fire — adding noise to a clean SOAP-test report would - defeat the purpose. Pin that the absence is intentional, not - an accident of test setup.""" - from flightcheck.checks.workday import _check_workflows - - # Empty workspace → discovery returns [] → no trailer. - (tmp_path / "workspace" / "agents").mkdir(parents=True, exist_ok=True) - - @dataclass - class R: - config: dict = field(default_factory=dict) - - runner = R() - results = _check_workflows(runner) +def _run(runner: _Runner): + results = _check_custom_workflow_inventory(runner) + assert len(results) == 1 + assert results[0].checkpoint_id == "WD-WF-CAT-001" + return results[0] + + +def test_workday_topics_present_pass_and_enumerate_components(): + r = _run(_runner_with_components([ + "msdyn_copilotforemployeeselfservicehr.topic.WorkdayAbsenceBalance", + "msdyn_copilotforemployeeselfservicehr.topic.WorkdayUpdatePhoneNumber", + "msdyn_copilotforemployeeselfservicehr.topic.Greeting", + ])) + + assert r.status == Status.PASSED.value + assert "2 Workday topic component(s)" in r.result + assert "WorkdayAbsenceBalance" in r.result + assert "WorkdayUpdatePhoneNumber" in r.result + assert "Greeting" not in r.result + assert "US 7792327" in r.result + assert r.remediation == "" + + +def test_workday_topics_absent_fails_with_repair_path(): + r = _run(_runner_with_components([ + "msdyn_copilotforemployeeselfservicehr.topic.Greeting", + "msdyn_copilotforemployeeselfservicehr.variable.PhoneLookupTable", + ])) + + assert r.status == Status.FAILED.value + assert "0 schemaName values matching" in r.result + assert ".topic.Workday*" in r.result + assert "Install or repair the Workday Declarative Agent extension" in ( + r.remediation + ) - trailer = [r for r in results if r.checkpoint_id == "WD-WF-CAT-LINK"] - assert len(trailer) == 0, ( - f"Trailer fired on clean workspace — should only fire when " - f"unknowns exist. Got: {[(r.checkpoint_id, r.result) for r in trailer]}" - ) +def test_empty_components_fail(): + r = _run(_runner_with_components([])) -# ───────────────────────────────────────────────────────────────────────── -# Catalog source resolution (Dataverse-only) -# -# The OOTB catalog is resolved by `_get_workday_ootb_catalog(runner)`: -# a single Dataverse query against -# `msdyn_employeeselfservicetemplateconfigs` filtered by -# `ismanaged=true`. There is NO fallback — when the catalog cannot be -# resolved, AGENTS.md principle #1 requires SKIPPED (no token) or -# WARNING (query error) instead of a misleading PASSED. These tests -# pin every leg of that decision matrix so a future refactor that -# re-introduces a silent fallback fails CI. -# -# Per tests/AGENTS.md: Dataverse is the `documented` tier — no cassette -# required, mock is built from MS Learn-documented response shape via -# `tests.mocks.dataverse`. `require_validated_mock(dv)` at module top -# enforces this can never silently downgrade to placeholder. -# ───────────────────────────────────────────────────────────────────────── + assert r.status == Status.FAILED.value + assert "0 schemaName values" in r.result + assert "re-run FlightCheck" in r.remediation -@dataclass -class _RunnerWithDataverse: - """Runner that exposes env_url + dv_token so `_get_workday_ootb_catalog` - follows the Dataverse leg (default-skipped on `_MinimalRunner`).""" - env_url: str = FAKE_DATAVERSE_URL - dv_token: str = FAKE_TOKEN - config: dict[str, Any] = field(default_factory=dict) - - -def _template_config_row(*, name: str, ismanaged: bool) -> dict[str, Any]: - """Build one `msdyn_employeeselfservicetemplateconfigs` record matching - the `msdyn_name,ismanaged` select projection the production helper - requests. Shape sourced from MS Learn Web API reference (the - `tests.mocks.dataverse` module is `documented` tier — see its - `MOCK_STATUS`).""" - return { - "@odata.etag": 'W/"1"', - "msdyn_name": name, - "ismanaged": ismanaged, - } - - -def _register_template_configs_response( - *, - base_url: str = FAKE_DATAVERSE_URL, - rows: list[dict[str, Any]] | None = None, - status: int = 200, -) -> None: - """Register a `responses` mock for the template-configs query. URL is - path-only (no query string) so it matches regardless of the exact - $select / $filter / paging params the production code builds.""" - payload = dv.collection(rows or []) if status == 200 else {"error": {"message": "mock failure"}} - responses.add( - method="GET", - url=f"{base_url}/api/data/v9.2/msdyn_employeeselfservicetemplateconfigs", - json=payload, - status=status, +def test_no_client_or_bot_id_skips(): + no_client = _run(_Runner(agentbuilder=None)) + assert no_client.status == Status.SKIPPED.value + assert "AgentBuilder client or active-agent botId not available" in ( + no_client.result ) + assert "Run /setup" in no_client.remediation + no_bot = _run(_Runner(config={}, agentbuilder=_FakeAgentBuilder({}))) + assert no_bot.status == Status.SKIPPED.value + assert "AgentBuilder client or active-agent botId not available" in ( + no_bot.result + ) + assert "active agent botId" in no_bot.remediation -class TestCatalogSource: - """Pin Dataverse-only semantics of `_get_workday_ootb_catalog`. - The four tests below cover every leg of the resolver's decision - matrix: (a) Dataverse-success with managed rows → catalog used, - (b) Dataverse-success with NO managed rows → check still runs (no - implicit PASS) and falls through to MANUAL because the - topic-referenced scenario is not in the managed set, (c) no - Dataverse token → SKIPPED (cannot validate without the catalog, - per AGENTS.md principle #1), (d) Dataverse query errors → - WARNING with the error message surfaced verbatim (per principle - #3 — fail loudly on API errors).""" - - @pytest.fixture(autouse=True) - def _isolate_env( - self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path - ) -> None: - monkeypatch.chdir(tmp_path) - - @responses.activate - def test_dataverse_managed_rows_used_as_catalog( - self, tmp_path: Path - ) -> None: - """When Dataverse is reachable and returns a managed - (ismanaged=true) row whose `msdyn_name` matches a topic's - scenarioName, the scenario is treated as OOTB and the check - PASSES. The result text MUST name Dataverse + ismanaged=true - as the source so an operator inspecting a green report knows - the catalog was tenant-accurate.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - custom_scenario = "msdyn_HRWorkdayTenantSpecificScenario_XYZ" - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "tenant-scenario.mcs.yml", - scenario_name=custom_scenario, - ) - - # Tenant has this scenario installed as a managed template - # config — i.e. it ships in the customer's Workday extension - # pack. Without the live-Dataverse leg, this would surface - # as MANUAL (no JSON seed exists to fall back to). - _register_template_configs_response(rows=[ - _template_config_row(name=custom_scenario, ismanaged=True), - ]) - - runner = _RunnerWithDataverse() - results = _check_custom_workflow_inventory(runner) - - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Passed", ( - f"Dataverse-managed scenario must surface as PASSED — " - f"the Dataverse-resolution leg regressed. " - f"Got {r.status!r}: {r.result!r}" - ) - # Source attribution pinned per principle #8 (`result` = - # observed). Operators must know the catalog was tenant-live. - assert "Dataverse" in r.result - assert "ismanaged=true" in r.result - # Cache populated with status "ok" for re-reads by the trailer. - assert runner._workday_ootb_catalog_cache[1] == "ok" - assert custom_scenario in runner._workday_ootb_catalog_cache[0] - - @responses.activate - def test_dataverse_unmanaged_rows_excluded_from_catalog( - self, tmp_path: Path - ) -> None: - """`ismanaged=false` rows are customer-added template configs, - NOT shipped by the extension pack. They must NOT count as OOTB - — otherwise a customer who added a custom scenario via /create - would see it falsely treated as "validated by Microsoft" and - skip the MANUAL checklist that exists to catch payload / - auth / test-prompt gaps. Pin the filter explicitly.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - custom_scenario = "msdyn_HRWorkdayCustomerAuthored_ABC" - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "customer-auth.mcs.yml", - scenario_name=custom_scenario, - ) - - # Same name exists in Dataverse but ismanaged=false → customer- - # authored, not OOTB. Must NOT short-circuit the MANUAL row. - _register_template_configs_response(rows=[ - _template_config_row(name=custom_scenario, ismanaged=False), - ]) - - runner = _RunnerWithDataverse() - results = _check_custom_workflow_inventory(runner) - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Manual", ( - f"ismanaged=false rows must NOT count as OOTB — " - f"the customer-added scenario should surface as MANUAL " - f"for review. Got {r.status!r}: result={r.result!r}" - ) - assert custom_scenario in r.result - assert "ISU account" in r.remediation # full checklist still emitted - # Cache: Dataverse WAS reached (status "ok"), it just returned - # no managed rows. The empty catalog correctly excludes the - # unmanaged scenario. - assert runner._workday_ootb_catalog_cache[1] == "ok" - assert custom_scenario not in runner._workday_ootb_catalog_cache[0] - - @responses.activate - def test_dataverse_query_failure_emits_warning( - self, tmp_path: Path - ) -> None: - """When the Dataverse query errors (500, network issue, expired - token, etc.), the check MUST emit a WARNING that surfaces the - error rather than silently passing — per AGENTS.md principle - #3 ("fail loudly on API errors, never silently swallow them - as PASS"). There is no JSON fallback: a tenant-accurate - catalog is the only valid source of truth, and a query error - means we don't have one.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "some-scenario.mcs.yml", - scenario_name="msdyn_AnyScenario_DoesntMatter", - ) +def test_malformed_bot_component_changes_warns(): + r = _run(_Runner( + agentbuilder=_FakeAgentBuilder({"botComponentChanges": {"bad": "shape"}}) + )) - # Dataverse query 500s — must surface as WARNING, not PASSED - # and not MANUAL (we cannot tell if it's custom). - _register_template_configs_response(status=500) + assert r.status == Status.WARNING.value + assert "invalid botComponentChanges" in r.result + assert "report the checkpoint ID (WD-WF-CAT-001)" in r.remediation - runner = _RunnerWithDataverse() - results = _check_custom_workflow_inventory(runner) - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Warning", ( - f"Dataverse query failure must emit WARNING per " - f"AGENTS.md principle #3, got {r.status!r}: {r.result!r}" - ) - # Result must name the failure mode so the operator can - # diagnose. Don't pin the exact error string (it comes from - # auth.query_all and may evolve) but pin the structural cues. - assert "Dataverse" in r.result - assert "msdyn_employeeselfservicetemplateconfigs" in r.result - assert "failed" in r.result.lower() or "error" in r.result.lower() - # Remediation must direct the operator to fix the underlying - # Dataverse problem (not work around it). - assert "Dataverse" in r.remediation - # Cache populated with the error status so re-reads don't - # re-query. - catalog, status_code = runner._workday_ootb_catalog_cache - assert catalog is None - assert status_code.startswith("query_error:") - # Unknown cache emptied so the trailer self-suppresses (the - # WARNING row is the single source of truth for this state). - assert runner._workday_unknown_scenarios == [] - - def test_no_dataverse_token_skips(self, tmp_path: Path) -> None: - """The CI / offline / no-auth runner has no env_url + dv_token. - The resolver MUST short-circuit to SKIPPED without attempting - any HTTP call (so this test doesn't even need - @responses.activate — any HTTP attempt would surface as a - connection error against an unrouted host). Per AGENTS.md - principle #1: without the catalog we cannot validate, so we - must not return PASSED.""" - from flightcheck.checks.workday import _check_custom_workflow_inventory - - agent_dir = tmp_path / "workspace" / "agents" / "ess-hr" - _write_topic_system_common_execution( - agent_dir / "topics" / "offline.mcs.yml", - scenario_name="msdyn_AnyScenario_DoesntMatter", - ) +def test_simplified_install_still_skips_before_api_read(): + runner = _Runner( + agentbuilder=_FakeAgentBuilder({"botComponentChanges": {"bad": "shape"}}), + _workday_package_flavor="simplified", + ) - # `_MinimalRunner` has no env_url / dv_token — Dataverse path - # must short-circuit before any HTTP call. - runner = _MinimalRunner() - results = _check_custom_workflow_inventory(runner) + r = _run(runner) - r = _result_by_id(results, "WD-WF-CAT-001") - assert r.status == "Skipped", ( - f"No-Dataverse-token runner must SKIP per AGENTS.md " - f"principle #1, got {r.status!r}: {r.result!r}" - ) - # Result must explain WHY this skipped (so the operator - # doesn't conflate it with the "Workday not wired up" SKIP). - assert "Dataverse" in r.result - assert "credentials" in r.result.lower() or "token" in r.result.lower() - # Remediation directs the operator at /setup so credentials get - # cached on the runner. - assert "/setup" in r.remediation or "Dataverse" in r.remediation - # Cache: catalog None with "no_token" status code. - catalog, status_code = runner._workday_ootb_catalog_cache - assert catalog is None - assert status_code == "no_token" - # Unknown cache emptied so the trailer self-suppresses. - assert runner._workday_unknown_scenarios == [] + assert r.status == Status.SKIPPED.value + assert "simplified" in r.result.lower() + assert "WD-PKG-001" in r.result diff --git a/tests/flightcheck/checks/test_workday_extension.py b/tests/flightcheck/checks/test_workday_extension.py index 0e641716b..3a4b2dfa1 100644 --- a/tests/flightcheck/checks/test_workday_extension.py +++ b/tests/flightcheck/checks/test_workday_extension.py @@ -10,9 +10,9 @@ connection, degrades gracefully when it does not. Cached-ref read + a best-effort Power Platform admin owner echo — no cassette required (the admin connections listing is the ``validated`` pp_admin mock). - * DV-CONN-001 — PASS/FAIL/NOT_CONFIGURED/SKIPPED over a documented-tier - Dataverse ``connectionreferences`` read (stubbed with ``responses``); owner - echo via the ``validated`` pp_admin mock. + * DV-CONN-001 — PASS/FAIL/SKIPPED over the validated minimalBots components + read (Workday SOAP connection reference; faked ``runner.agentbuilder``); + owner echo via the ``validated`` pp_admin mock. * WD-REST-001 — pure-config check (restBaseUrl trimmed to '/api'). * WD-REST-002 — pure local-file check (user-context redirect topic); SKIPPED on the legacy install path. @@ -28,22 +28,18 @@ from dataclasses import dataclass, field from typing import Any -import responses - from tests.conftest import require_validated_mock +from tests.mocks import agentbuilder_connectivity as ab from tests.mocks import dataverse as dv from tests.mocks import pp_admin as pp +require_validated_mock(ab) require_validated_mock(dv) require_validated_mock(pp) from flightcheck.checks import workday_extension as wx # noqa: E402 from flightcheck.runner import Priority, Role, Status # noqa: E402 -_DV_CONNECTOR_ID = ( - "/providers/Microsoft.PowerApps/apis/shared_commondataserviceforapps" -) - # ───────────────────────────────────────────────────────────────────── # Minimal runner. The emitters read only these attributes; anything the @@ -62,12 +58,24 @@ def get_connections(self, _env_id: str): return self._connections +class _FakeAgentBuilder: + """Stand-in for FlightCheckRunner.agentbuilder. Only ``fetch_components`` + is consumed (DV-CONN-001's connection-reference read).""" + + def __init__(self, components: dict[str, Any]): + self._components = components + + def fetch_components(self, _agent_id: str): + return self._components + + @dataclass class _Runner: config: Any = field(default_factory=dict) env_url: str | None = None dv_token: str | None = None pp_admin: Any = None + agentbuilder: Any = None env_id: str | None = None _workday_connection_refs: list[dict[str, Any]] = field(default_factory=list) @@ -88,28 +96,6 @@ def _by_id(results): return {r.checkpoint_id: r for r in results} -def _dv_ref(*, connection_id, statuscode=1): - """A Dataverse connection reference matching the extension pack's shipped - ref (connector shared_commondataserviceforapps, logical-name suffix - 92b66).""" - return dv.connection_ref( - logical_name="msdyn_sharedcommondataserviceforapps_92b66", - display_name="Microsoft Dataverse", - connector_id=_DV_CONNECTOR_ID, - connection_id=connection_id, - statuscode=statuscode, - ) - - -def _register_refs(base_url: str, refs: list[dict[str, Any]]) -> None: - responses.add( - method="GET", - url=f"{base_url}/api/data/v9.2/connectionreferences", - json=dv.collection(refs), - status=200, - ) - - # ───────────────────────────────────────────────────────────────────── # WD-CONN-AUTH-001 — always MANUAL echo (S5.3). # ───────────────────────────────────────────────────────────────────── @@ -253,147 +239,111 @@ def test_never_passes_regardless_of_state(self): # ───────────────────────────────────────────────────────────────────── -# DV-CONN-001 — Dataverse connection binding (S5.4, PASS/FAIL). +# DV-CONN-001 — Workday SOAP connection binding (S5.4, PASS/FAIL). # ───────────────────────────────────────────────────────────────────── +def _runner_with_refs(references, *, pp_admin=None, env_id=None): + """A runner whose faked ``agentbuilder.fetch_components`` returns the given + connection references and whose config names an active agent (botId).""" + components = ab.components_with_references(references=references) + return _Runner( + config={"agent": {"botId": ab.MOCK_AGENT_ID}}, + agentbuilder=_FakeAgentBuilder(components), + pp_admin=pp_admin, + env_id=env_id, + ) + + class TestDataverseConnection: - @responses.activate - def test_bound_active_with_owner_echo_passes( - self, fake_dataverse_url, fake_token - ): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-active", statuscode=1)], - ) + def test_bound_with_owner_echo_passes(self): owner_conn = pp.connection( - name="dv-conn-active", - api_name="shared_commondataserviceforapps", + name="wd-conn-active", + api_name="shared_workdaysoap", extra_properties={"accountName": "maker@contoso.com"}, ) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id="wd-conn-active")], pp_admin=_FakePPAdmin([owner_conn]), env_id="env-1", ) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.PASSED.value - assert "bound to an active" in r.result + assert "bound to a connection" in r.result assert "maker@contoso.com" in r.result assert "your own account" in r.result - @responses.activate - def test_passes_without_pp_admin_notes_owner_unreadable( - self, fake_dataverse_url, fake_token - ): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-active", statuscode=1)], + def test_passes_without_pp_admin_notes_owner_unreadable(self): + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id="wd-conn-active")], ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.PASSED.value assert "owner could not be read" in r.result assert "your own account" in r.result - @responses.activate - def test_runtime_dataverse_reference_passes( - self, fake_dataverse_url, fake_token - ): - runtime_ref = dv.workday_connection_refs_runtime()[1] - _register_refs(fake_dataverse_url, [runtime_ref]) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, - ) - - r = _by_id( - wx.run_workday_extension_checks(runner) - )["DV-CONN-001"] - - assert r.status == Status.PASSED.value - assert ( - "msdyn_sharedcommondataserviceforapps_workdayruntime" - in r.result - ) - - @responses.activate - def test_mixed_runtime_and_legacy_dataverse_refs_warn( - self, fake_dataverse_url, fake_token - ): - runtime_ref = dv.workday_connection_refs_runtime()[1] - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="legacy-dv"), runtime_ref], - ) - runner = _Runner( - env_url=fake_dataverse_url, - dv_token=fake_token, + def test_unbound_fails(self): + runner = _runner_with_refs( + [ab.workday_connection_reference(connection_id=None)], ) - r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.WARNING.value - assert "Multiple ESS Dataverse connection references" in r.result - assert "Remove obsolete Workday package references" in r.remediation + assert r.status == Status.FAILED.value + assert "unbound" in r.result + assert "connectionId=null" in r.result + assert "bind the Workday SOAP connection reference" in r.remediation - @responses.activate - def test_unbound_fails(self, fake_dataverse_url, fake_token): - _register_refs( - fake_dataverse_url, [_dv_ref(connection_id=None, statuscode=1)] + def test_workday_ref_absent_fails(self): + # Only a ServiceNow ref is present - no Workday SOAP ref. + runner = _runner_with_refs( + [ + ab.connection_reference_change( + connector="shared_service-now", + connection_id="sn-1", + ) + ] ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] assert r.status == Status.FAILED.value - assert "unbound" in r.result - assert "connectionid=null" in r.result - assert "bind the Dataverse connection reference" in r.remediation - - @responses.activate - def test_inactive_statuscode_fails(self, fake_dataverse_url, fake_token): - _register_refs( - fake_dataverse_url, - [_dv_ref(connection_id="dv-conn-inactive", statuscode=2)], - ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) + assert "was not found" in r.result + assert "shared_workdaysoap" in r.result + assert "Install or repair the Workday extension pack" in r.remediation + + def test_no_agentbuilder_client_skips(self): + runner = _Runner(config={"agent": {"botId": ab.MOCK_AGENT_ID}}) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.FAILED.value - assert "inactive" in r.result - assert "statuscode=2" in r.result - assert "Re-authenticate or re-bind" in r.remediation - - @responses.activate - def test_missing_ref_not_configured(self, fake_dataverse_url, fake_token): - # Only a Workday ref present — no Dataverse (92b66) ref. - _register_refs( - fake_dataverse_url, - [ - dv.connection_ref( - logical_name="new_sharedworkdaysoap_ff0df", - display_name="OAuthUser", - connector_id=dv.WORKDAY_SOAP_CONNECTOR_ID, - connection_id="wd-conn-1", - ) - ], + assert r.status == Status.SKIPPED.value + assert "not available" in r.result + + def test_no_active_agent_botid_skips(self): + runner = _Runner( + config={}, + agentbuilder=_FakeAgentBuilder(ab.components_with_references()), ) - runner = _Runner(env_url=fake_dataverse_url, dv_token=fake_token) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.NOT_CONFIGURED.value - assert "was not found in this environment" in r.result - assert "Install/repair the Workday extension pack" in r.remediation + assert r.status == Status.SKIPPED.value + assert "not available" in r.result - def test_no_dv_token_skips(self): - runner = _Runner(env_url="https://x.crm.dynamics.com", dv_token="") + def test_malformed_changeset_degrades_to_warning(self): + # A 200 payload whose connectionReferenceChanges is present but not a + # list is a shape we do not understand: fail loudly (dispatcher WARNING) + # rather than reporting a confident "reference not found" FAILED. + runner = _Runner( + config={"agent": {"botId": ab.MOCK_AGENT_ID}}, + agentbuilder=_FakeAgentBuilder( + {"connectionReferenceChanges": {"unexpected": "dict"}} + ), + ) r = _by_id(wx.run_workday_extension_checks(runner))["DV-CONN-001"] - assert r.status == Status.SKIPPED.value - assert "Dataverse token not available" in r.result + assert r.status == Status.WARNING.value + assert "Unable to run DV-CONN-001" in r.result + assert "DV-CONN-001" in r.remediation # ───────────────────────────────────────────────────────────────────── diff --git a/tests/flightcheck/checks/test_workday_reference_data.py b/tests/flightcheck/checks/test_workday_reference_data.py index 76582afe2..94390ecf1 100644 --- a/tests/flightcheck/checks/test_workday_reference_data.py +++ b/tests/flightcheck/checks/test_workday_reference_data.py @@ -1,197 +1,129 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Tests for WD-REF-001 (Workday write-scenario reference-data availability). - -The check reconciles the reference picklists each Workday topic REQUESTS from -the shared GetReferenceData topic (``referenceDataKey: KEY``) against the keys -GetReferenceData SUPPORTS (its ``referenceDataKey = "KEY"`` switch). Both come -from Dataverse ``botcomponents`` via ``query_all`` (documented tier — stubbed -here). The request-template ID types are deliberately NOT used (they don't map -1:1 to GetReferenceData keys and would false-positive on OOTB scenarios — see -the WD-REF-001 comment in checks/workday.py). -""" +"""Tests for WD-REF-001 DA reference-data component inventory.""" from __future__ import annotations -from types import SimpleNamespace +from dataclasses import dataclass, field +from typing import Any +from tests.conftest import require_validated_mock +from tests.mocks import agentbuilder_connectivity as ab -# ── pure-logic extractor tests (no network) ─────────────────────────────── +require_validated_mock(ab) -def test_extract_supported_keys_reads_the_switch(): - from flightcheck.checks.workday import _extract_supported_reference_keys +from flightcheck.checks.workday import _check_workday_reference_data # noqa: E402 +from flightcheck.runner import Status # noqa: E402 - data = ( - 'condition: =Topic.referenceDataKey = "Phone_Device_Type_ID"\n' - "condition: =Topic.referenceDataKey = 'Related_Person_Relationship_ID'\n" - # input declaration (no value on the line) must NOT count as supported - "referenceDataKey:\n displayName: referenceDataKey\n" - ) - assert _extract_supported_reference_keys(data) == { - "Phone_Device_Type_ID", "Related_Person_Relationship_ID", - } +class _FakeAgentBuilder: + def __init__(self, payload: dict[str, Any]): + self._payload = payload + + def fetch_components(self, _agent_id: str) -> dict[str, Any]: + return self._payload -def test_extract_requested_keys_reads_literal_call_inputs(): - from flightcheck.checks.workday import _extract_requested_reference_keys - data = ( - "referenceDataKey: Phone_Device_Type_ID\n" - "referenceDataKey: Country_Phone_Code_ID\n" - # a Power Fx expression value is not a static key -> not matched - "referenceDataKey: =SomeDynamic(Expr)\n" - # the switch form (=) is a SUPPORTED marker, not a request -> not matched - 'condition: =Topic.referenceDataKey = "Marital_Status_ID"\n' +@dataclass +class _Runner: + config: dict[str, Any] = field( + default_factory=lambda: {"agent": {"botId": ab.MOCK_AGENT_ID}} ) - assert _extract_requested_reference_keys(data) == { - "Phone_Device_Type_ID", "Country_Phone_Code_ID", - } - - -# ── integration tests (stubbed query_all) ───────────────────────────────── - -_GETREF = { - "name": "Workday System Get ReferenceData", - "schemaname": "msdyn_copilotforemployeeselfservicehr.topic.GetReferenceData", - "data": ( - 'condition: =Topic.referenceDataKey = "Phone_Device_Type_ID"\n' - 'condition: =Topic.referenceDataKey = "Country_Phone_Code_ID"\n' - 'condition: =Topic.referenceDataKey = "Related_Person_Relationship_ID"\n' - "referenceDataKey:\n displayName: referenceDataKey\n" - ), -} -_PHONE = { - "name": "Workday Update PhoneNumber", - "schemaname": "msdyn_copilotforemployeeselfservicehr.topic.EmployeeUpdatePhoneNumber", - "data": "referenceDataKey: Phone_Device_Type_ID\nreferenceDataKey: Country_Phone_Code_ID\n", -} -_DEPENDENT = { - "name": "Workday Add Dependent", - "schemaname": "msdyn_copilotforemployeeselfservicehr.topic.WorkdayAddDependent", - "data": "referenceDataKey: Related_Person_Relationship_ID\n", -} -_BAD = { - "name": "Workday Custom Marital Status", - "schemaname": "msdyn_copilotforemployeeselfservicehr.topic.CustomMaritalStatus", - "data": "referenceDataKey: Marital_Status_ID\n", # NOT in _GETREF's switch -} - - -def _runner(): - return SimpleNamespace(env_url="https://org.crm.dynamics.com", dv_token="t") - - -def _stub(monkeypatch, topics): - import auth - monkeypatch.setattr(auth, "query_all", lambda *a, **kw: list(topics)) - - -def _run(): - from flightcheck.checks.workday import _check_workday_reference_data - results = _check_workday_reference_data(_runner()) + agentbuilder: Any = None + + +def _runner_with_components(schema_names: list[str]) -> _Runner: + return _Runner( + agentbuilder=_FakeAgentBuilder( + ab.components_with_bot_components( + bot_components=[ + ab.bot_component_change(schema_name=name) + for name in schema_names + ] + ) + ) + ) + + +def _run(runner: _Runner): + results = _check_workday_reference_data(runner) assert len(results) == 1 assert results[0].checkpoint_id == "WD-REF-001" return results[0] -def test_all_requested_keys_supported_passes(monkeypatch): - _stub(monkeypatch, [_GETREF, _PHONE, _DEPENDENT]) - r = _run() - assert r.status == "Passed" - assert "request only keys GetReferenceData supports" in r.result +def test_lookup_tables_and_reference_topics_present_pass(): + r = _run(_runner_with_components([ + "msdyn_copilotforemployeeselfservicehr.variable.PhoneLookupTable", + "msdyn_copilotforemployeeselfservicehr.variable.CountryLookupTable", + "msdyn_copilotforemployeeselfservicehr.topic.WorkdaySystemGetReferenceData", + ( + "msdyn_copilotforemployeeselfservicehr.topic." + "WorkdaySystemRefreshReferenceData" + ), + ])) + + assert r.status == Status.PASSED.value + assert "2 Workday LookupTable variable component(s)" in r.result + assert "reference-data topic component(s)" in r.result + assert "US 7792327" in r.result assert r.remediation == "" -def test_topic_requesting_unsupported_key_fails(monkeypatch): - _stub(monkeypatch, [_GETREF, _PHONE, _BAD]) - r = _run() - assert r.status == "Failed" - assert "1 of 2 Workday topic(s)" in r.result - assert "Workday Custom Marital Status" in r.result - assert "Marital_Status_ID" in r.result - # The other (valid) topic must NOT be reported as a gap. - assert "Update PhoneNumber" not in r.result - assert "GetReferenceData" in r.remediation - # A clickable fix-link to Copilot Studio is present in the remediation. - assert "copilotstudio.microsoft.com" in r.remediation - assert "](" in r.remediation # markdown link - - -def test_failure_remediation_contains_resolved_studio_deeplink(monkeypatch): - # A runner whose parsed config carries env_id + agents[] (the real shape - # produced by setup.py / cli.py) must yield a *resolved* deep link to the - # agent's overview page — not the generic homepage fallback. - _stub(monkeypatch, [_GETREF, _PHONE, _BAD]) - from flightcheck.checks.workday import _check_workday_reference_data - - runner = SimpleNamespace( - env_url="https://org.crm.dynamics.com", - dv_token="t", - env_id="ENV-123", - config={ - "activeAgent": "esshrwdayonlyoauth", - "agents": [{"slug": "esshrwdayonlyoauth", "botId": "BOT-456"}], - }, - ) - r = _check_workday_reference_data(runner)[0] - assert r.status == "Failed" - assert ( - "/environments/ENV-123/bots/BOT-456/overview" in r.remediation - ), r.remediation - - -def test_deeplink_resolves_from_agents_when_activeagent_absent(monkeypatch): - # Defensive fallback: even without activeAgent/agent.slug, the first - # agents[] entry resolves the deep link (covers the silent-homepage path - # raised in review). - _stub(monkeypatch, [_GETREF, _PHONE, _BAD]) - from flightcheck.checks.workday import _check_workday_reference_data - - runner = SimpleNamespace( - env_url="https://org.crm.dynamics.com", - dv_token="t", - env_id="ENV-123", - config={"agents": [{"slug": "esshrwdayonlyoauth", "botId": "BOT-456"}]}, - ) - r = _check_workday_reference_data(runner)[0] - assert r.status == "Failed" - assert "/environments/ENV-123/bots/BOT-456/overview" in r.remediation +def test_missing_lookup_tables_fail_with_repair_path(): + r = _run(_runner_with_components([ + "msdyn_copilotforemployeeselfservicehr.topic.WorkdaySystemGetReferenceData", + ( + "msdyn_copilotforemployeeselfservicehr.topic." + "WorkdaySystemRefreshReferenceData" + ), + ])) + + assert r.status == Status.FAILED.value + assert "0 .variable.*LookupTable component schemaName(s)" in r.result + assert "LookupTable variables" in r.remediation + assert "re-import" in r.remediation + + +def test_missing_reference_topics_fail_with_repair_path(): + r = _run(_runner_with_components([ + "msdyn_copilotforemployeeselfservicehr.variable.PhoneLookupTable", + ])) + assert r.status == Status.FAILED.value + assert "missing reference-data topic component schemaName(s)" in r.result + assert "WorkdaySystemGetReferenceData" in r.result + assert "WorkdaySystemRefreshReferenceData" in r.result + assert "WorkdaySystemGetReferenceData" in r.remediation -def test_getreferencedata_missing_fails(monkeypatch): - _stub(monkeypatch, [_PHONE, _DEPENDENT]) # no GetReferenceData topic - r = _run() - assert r.status == "Failed" - assert "'GetReferenceData' topic is not installed" in r.result - assert "Install/repair the Workday extension" in r.remediation +def test_empty_components_fail(): + r = _run(_runner_with_components([])) -def test_no_topic_requests_reference_data_is_not_configured(monkeypatch): - # Only GetReferenceData present; its own declaration/switch is not a request. - _stub(monkeypatch, [_GETREF]) - r = _run() - assert r.status == "NotConfigured" - assert "No Workday topic requests a reference-data picklist" in r.result + assert r.status == Status.FAILED.value + assert "0 .variable.*LookupTable" in r.result + assert "missing reference-data topic" in r.result + assert "repair or re-import" in r.remediation -def test_missing_dataverse_token_is_skipped(monkeypatch): - from flightcheck.checks.workday import _check_workday_reference_data - r = _check_workday_reference_data( - SimpleNamespace(env_url="https://org.crm.dynamics.com", dv_token=None) - )[0] - assert r.status == "Skipped" - assert "Dataverse token not available" in r.result +def test_no_client_or_bot_id_skips(): + no_client = _run(_Runner(agentbuilder=None)) + assert no_client.status == Status.SKIPPED.value + assert "AgentBuilder client or active-agent botId not available" in no_client.result + assert "Run /setup" in no_client.remediation + no_bot = _run(_Runner(config={}, agentbuilder=_FakeAgentBuilder({}))) + assert no_bot.status == Status.SKIPPED.value + assert "AgentBuilder client or active-agent botId not available" in no_bot.result + assert "active agent botId" in no_bot.remediation -def test_query_error_is_skipped(monkeypatch): - import auth - def _boom(*a, **kw): - raise RuntimeError("403 Forbidden") +def test_malformed_bot_component_changes_warns(): + r = _run(_Runner( + agentbuilder=_FakeAgentBuilder({"botComponentChanges": {"bad": "shape"}}) + )) - monkeypatch.setattr(auth, "query_all", _boom) - r = _run() - assert r.status == "Skipped" - assert "Unable to read Dataverse topic configuration" in r.result - assert "403 Forbidden" in r.result + assert r.status == Status.WARNING.value + assert "invalid botComponentChanges" in r.result + assert "report the checkpoint ID (WD-REF-001)" in r.remediation diff --git a/tests/flightcheck/test_registry.py b/tests/flightcheck/test_registry.py index e8882af57..86fdc3718 100644 --- a/tests/flightcheck/test_registry.py +++ b/tests/flightcheck/test_registry.py @@ -294,7 +294,7 @@ class TestWorkdayExtensionCheckpoints: """skill-5 mints five checkpoints, all sharing checks/workday_extension.run_workday_extension_checks, category "Workday Extension". Two are always-MANUAL echoes/attestations, three are - programmatic (one Dataverse read + two pure-local).""" + programmatic (one minimalBots components read + two pure-local).""" _ALL = ( "WD-CONN-AUTH-001", @@ -329,10 +329,12 @@ def test_conn_auth_exact_beats_wd_conn_family(self): assert registry.resolve("WD-CONN-AUTH-001").key == "WD-CONN-AUTH-001" assert registry.resolve("WD-CONN-AUTH-001").is_family is False - def test_dv_conn_spec_declares_dataverse_and_pp_admin(self): + def test_dv_conn_spec_declares_agentbuilder_and_pp_admin(self): spec = registry.resolve("DV-CONN-001") - assert spec.clients == frozenset({registry.DATAVERSE, registry.PP_ADMIN}) - assert spec.requires_dataverse_endpoint is True + assert spec.clients == frozenset( + {registry.AGENTBUILDER, registry.PP_ADMIN} + ) + assert spec.requires_dataverse_endpoint is False assert spec.prereqs == () assert Role.ESS_MAKER.value in spec.roles @@ -354,7 +356,7 @@ def test_net_check_is_clientless_and_ppadmin_gated(self): def test_dv_conn_plan_unions_clients(self): plan = registry.transitive_requirements("DV-CONN-001") - assert registry.DATAVERSE in plan.clients + assert registry.AGENTBUILDER in plan.clients assert registry.PP_ADMIN in plan.clients def test_all_five_are_listable(self): @@ -363,6 +365,42 @@ def test_all_five_are_listable(self): assert cp in keys +class TestWorkdayDaComponentCheckpoints: + """DA Workday component checks are flowless AgentBuilder reads.""" + + def test_wd_ref_uses_agentbuilder_without_dataverse(self): + spec = registry.resolve("WD-REF-001") + assert spec is not None and spec.key == "WD-REF-001" + assert spec.category_label == "Workday" + assert spec.category_fn is run_workday_checks + assert spec.clients == frozenset({registry.AGENTBUILDER}) + assert spec.requires_dataverse_endpoint is False + assert Role.ESS_MAKER.value in spec.roles + assert Role.WORKDAY_ADMIN.value in spec.roles + + def test_wd_wf_cat_exact_entry_beats_legacy_wd_wf_family(self): + spec = registry.resolve("WD-WF-CAT-001") + assert spec is not None and spec.key == "WD-WF-CAT-001" + assert spec.category_label == "Workday" + assert spec.category_fn is run_workday_checks + assert spec.clients == frozenset({registry.AGENTBUILDER}) + assert spec.requires_dataverse_endpoint is False + assert Role.ESS_MAKER.value in spec.roles + + def test_component_plans_are_agentbuilder_only(self): + for checkpoint_id in ("WD-REF-001", "WD-WF-CAT-001"): + plan = registry.transitive_requirements(checkpoint_id) + assert plan.clients == frozenset({registry.AGENTBUILDER}) + assert plan.requires_config is True + assert plan.requires_dataverse_endpoint is False + assert [label for label, _ in plan.ordered_fns] == ["Workday"] + + def test_both_component_checks_are_listable(self): + keys = {spec.key for spec in registry.list_checkpoints()} + assert "WD-REF-001" in keys + assert "WD-WF-CAT-001" in keys + + class TestTopicCheckpoints: """skill-6 mints two FAMILY checkpoints (one row per new/custom topic), both sharing checks/topics.run_topic_checks, category "Workday Topics". diff --git a/tests/mocks/agentbuilder_connectivity.py b/tests/mocks/agentbuilder_connectivity.py index f666f928c..ea3f200aa 100644 --- a/tests/mocks/agentbuilder_connectivity.py +++ b/tests/mocks/agentbuilder_connectivity.py @@ -174,6 +174,40 @@ def shared_connection_parameters_json_string(**kwargs: Any) -> str: return json.dumps(shared_connection_parameters(**kwargs)) +def bot_component_change(*, schema_name: str) -> dict[str, Any]: + """One ``botComponentChanges`` entry in the validated minimalBots + components shape. + + Source (validated): + tests/fixtures/cassettes/agentbuilder_readiness.yaml covers + POST /copilotstudio/minimalBots/api/{agentId}/components with a + top-level ``botComponentChanges`` list. Captured payload variants in + this repo use either ``botComponent.name`` or ``component.schemaName``; + tests use the current ``botComponent.name`` shape consumed by + FlightCheck. + """ + return { + "changeType": "Insert", + "botComponent": { + "name": schema_name, + "componentType": 9, + "content": "{}", + }, + } + + +def components_with_bot_components( + *, + bot_components: Iterable[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """``components()`` with ``botComponentChanges`` replaced.""" + payload = components() + payload["botComponentChanges"] = ( + [] if bot_components is None else list(bot_components) + ) + return payload + + def components_with_references( *, references: Iterable[dict[str, Any]] | None = None,