diff --git a/solutions/ess-maker-skills/scripts/flightcheck/cli.py b/solutions/ess-maker-skills/scripts/flightcheck/cli.py index 426ad5330..50999d4fc 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/cli.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/cli.py @@ -41,6 +41,7 @@ from flightcheck.runner import ( FlightCheckRunner, + ValidationContext, save_results, Status, bucket_results, @@ -710,6 +711,25 @@ def _is_native_no_dataverse(config: dict, env_url: str) -> bool: return str(active.get("releaseLine") or "").casefold() == "da" +def _validation_context_from_args( + args, + config: dict, + env_url: str, + env_id: str | None, +) -> ValidationContext: + active = _active_agent_config(config) + return ValidationContext( + realm=getattr(args, "validation_realm", None) or config.get("realm", ""), + environment_id=env_id or config.get("environmentId", ""), + environment_url=env_url or config.get("dataverseEndpoint", ""), + agent_schema_name=( + getattr(args, "agent_schema_name", None) + or active.get("schemaName", "") + ), + agent_id=active.get("botId", ""), + ) + + def _resolve_environment_ring( config: dict, *, @@ -1159,6 +1179,231 @@ def _run_single_checkpoint(args): sys.exit(1 if result.failed > 0 or result.errors > 0 else 0) +def _run_profile(args): + """Run a named validation profile and emit the versioned result contract.""" + from flightcheck import registry + + profile_name = args.profile + profile = registry.resolve_profile(profile_name) + if profile is None: + print(f"ERROR: Unknown profile {profile_name!r}.") + print("Valid profiles:") + for item in registry.list_profiles(): + print(f" {item.name}") + sys.exit(2) + + plan = registry.profile_requirements(profile_name) + needed = plan.clients + + config = {} + config_path = os.path.join(".local", "config.json") + if os.path.exists(config_path): + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + elif plan.requires_config: + print("ERROR: .local/config.json not found. Run /setup first.") + sys.exit(1) + + env_url = args.environment_url or config.get("dataverseEndpoint", "") + if plan.requires_dataverse_endpoint and not env_url: + print("ERROR: No dataverseEndpoint in .local/config.json.") + sys.exit(1) + + quiet_auth = getattr(args, "quiet_auth", False) + if not quiet_auth: + print() + print("=" * 64) + print(" ESS FLIGHTCHECK — Validation Profile") + print("=" * 64) + print(f" Profile: {profile_name}") + if env_url: + print(f" Environment: {env_url}") + print(f" Clients: {', '.join(sorted(needed)) or '(none)'}") + print("=" * 64) + print() + + dv_token = None + tenant_id = None + graph = None + pp_admin = None + pva = None + powerplatform = None + agentbuilder = None + connectivity = None + + if needed & { + registry.GRAPH, + registry.PP_ADMIN, + registry.PVA, + registry.DATAVERSE, + registry.POWERPLATFORM, + }: + from auth import discover_tenant + if env_url: + try: + tenant_id = discover_tenant(env_url) + except Exception as e: + print(f" Tenant discovery: WARNING — {e}") + tenant_id = "organizations" + else: + tenant_id = "organizations" + + if registry.DATAVERSE in needed and env_url: + from auth import authenticate + if not quiet_auth: + print("Authenticating to Dataverse...") + try: + dv_token = authenticate(env_url) + if not quiet_auth: + print(" Dataverse: OK") + except Exception as e: + print(f" Dataverse: WARNING — {e}") + dv_token = None + + if registry.GRAPH in needed: + if not quiet_auth: + print("Authenticating to Microsoft Graph...") + graph = GraphClient(tenant_id) + try: + graph.authenticate() + if not quiet_auth: + print(" Graph: OK") + except Exception as e: + print(f" Graph: WARNING — {e}") + graph = None + + if registry.PP_ADMIN in needed: + if not quiet_auth: + print("Authenticating to Power Platform Admin API...") + pp_admin = PPAdminClient(tenant_id) + try: + pp_admin.authenticate() + if not quiet_auth: + print(" Power Platform: OK") + except Exception as e: + print(f" Power Platform: WARNING — {e}") + pp_admin = None + + env_id = args.environment_id or config.get("environmentId") or None + if not env_id and registry.PP_ADMIN in needed and env_url: + env_id = derive_environment_id(env_url, dv_token, pp_admin=pp_admin) + + if needed & {registry.AGENTBUILDER, registry.CONNECTIVITY}: + native_host = config.get("powerPlatformApiEndpoint", "") + if not native_host: + print( + "ERROR: No powerPlatformApiEndpoint in .local/config.json. " + "Run /setup again." + ) + sys.exit(1) + try: + native_ring = ring_from_environment_host(native_host) + native_host = validate_environment_host(native_host, native_ring) + except ValueError as e: + print(f"ERROR: {e}") + sys.exit(1) + if not quiet_auth: + print("Authenticating to native AgentBuilder APIs...") + try: + native_token, native_tenant_id = authenticate_flightcheck( + native_ring, + include_connectivity=registry.CONNECTIVITY in needed, + ) + tenant_id = native_tenant_id + agentbuilder = AgentBuilderClient( + native_host, + native_token, + ring=native_ring, + tenant_id=native_tenant_id, + api_version=config.get( + "agentBuilderApiVersion", "2024-10-01" + ), + ) + if registry.CONNECTIVITY in needed: + connectivity = ConnectivityClient( + native_token, + ring=native_ring, + api_version=config.get( + "agentBuilderApiVersion", "2024-10-01" + ), + ) + if not quiet_auth: + print(" Native AgentBuilder APIs: OK") + except Exception as e: + print(f" Native AgentBuilder APIs: WARNING — {e}") + agentbuilder = None + connectivity = None + + if registry.PVA in needed: + if not quiet_auth: + print("Authenticating to Copilot Studio (Island Gateway)...") + pva = PVAClient(tenant_id, env_url) + try: + pva.authenticate() + if not quiet_auth: + print(" Copilot Studio: OK") + except Exception as e: + print(f" Copilot Studio: WARNING — {e}") + pva = None + + if registry.POWERPLATFORM in needed: + if not quiet_auth: + print("Authenticating to Power Platform API (capacity allocation)...") + powerplatform = PowerPlatformClient(tenant_id) + try: + powerplatform.authenticate() + if not quiet_auth: + print(" Power Platform API: OK") + except Exception as e: + print(f" Power Platform API: WARNING — {e}") + powerplatform = None + + try: + validation_context = _validation_context_from_args( + args, config, env_url, env_id + ) + except ValueError as e: + print(f"ERROR: {e}") + print("Pass --validation-realm dev|test|prod (or set \"realm\" in " + ".local/config.json) for profile runs.") + sys.exit(1) + + runner = FlightCheckRunner( + scope=f"profile:{profile_name}", + target_matcher=lambda cid: registry.profile_matches(profile_name, cid), + ) + runner.config = config + runner.env_url = env_url + runner.dv_token = dv_token + runner.env_id = env_id + runner.graph = graph + runner.pp_admin = pp_admin + runner.pva = pva + runner.powerplatform = powerplatform + runner.azure_arm = None + runner.agentbuilder = agentbuilder + runner.connectivity = connectivity + + for label, fn in plan.ordered_fns: + runner.register(label, fn) + + if not quiet_auth: + print("\nRunning profile...\n") + result = runner.run() + result.profile = profile_name + result.profile_checkpoints = list(profile.checkpoint_ids) + result.validation_context = validation_context.to_dict() + + _print_prioritized_summary(result, verbose_manual=True) + save_results(result, args.output) + + if not result.results: + print(f"\nNOTE: profile {profile_name} produced no result rows.") + sys.exit(1) + + sys.exit(1 if result.failed > 0 or result.blocked > 0 or result.errors > 0 else 0) + + def main(): # Force UTF-8 console output so summary glyphs (→, •) don't crash on # Windows cp1252 terminals. Without this, _print_prioritized_summary @@ -1172,7 +1417,7 @@ def main(): parser.add_argument( "--scope", default=None, choices=["full"] + list(SCOPE_MAP.keys()), - help="Validation scope (default: full). Mutually exclusive with --checkpoint.", + help="Validation scope (default: full). Mutually exclusive with --checkpoint/--profile.", ) parser.add_argument( "--output", default="workspace/flightcheck", @@ -1203,7 +1448,24 @@ def main(): help="Run exactly one checkpoint (or a family, e.g. WD-FLOW-*) by ID and " "report only its result. Hydrates the checkpoint's declared " "prerequisites and initialises only the clients it needs. Mutually " - "exclusive with --scope.", + "exclusive with --scope/--profile.", + ) + parser.add_argument( + "--profile", + help="Run a named validation profile, e.g. workday-da:setup-readiness. " + "Profile runs need a realm (--validation-realm, or a \"realm\" key " + "in .local/config.json) and emit the versioned Connect result " + "contract in results.json.", + ) + parser.add_argument( + "--validation-realm", + choices=["dev", "test", "prod"], + help="Realm for a --profile validation context. Required with --profile " + "unless .local/config.json supplies a \"realm\" value.", + ) + parser.add_argument( + "--agent-schema-name", + help="Agent schema name for a --profile validation context.", ) parser.add_argument( "--connect-config", @@ -1294,7 +1556,7 @@ def main(): except ValueError as e: parser.error(f"invalid --agent-slug: {e}") - # --- Single-checkpoint mode (additive; leaves all --scope behavior intact) --- + # --- Targeted modes (additive; leave --scope behavior intact) --- if args.list_checkpoints: _print_checkpoint_list() sys.exit(0) @@ -1307,12 +1569,19 @@ def main(): sys.exit(0) if args.checkpoint: - if args.scope is not None: - print("ERROR: --checkpoint and --scope are mutually exclusive.") + if args.scope is not None or args.profile: + print("ERROR: --checkpoint is mutually exclusive with --scope/--profile.") sys.exit(2) _run_single_checkpoint(args) return # _run_single_checkpoint always exits; defensive only. + if args.profile: + if args.scope is not None: + print("ERROR: --profile and --scope are mutually exclusive.") + sys.exit(2) + _run_profile(args) + return # _run_profile always exits; defensive only. + # Normal scope mode: --scope defaults to "full" when omitted. (Default is # None on the parser so checkpoint-mode can detect an explicit --scope.) if args.scope is None: diff --git a/solutions/ess-maker-skills/scripts/flightcheck/registry.py b/solutions/ess-maker-skills/scripts/flightcheck/registry.py index dc759a9a1..68c4351f7 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/registry.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/registry.py @@ -159,6 +159,15 @@ class ResolvedPlan: ordered_fns: list = field(default_factory=list) +@dataclass(frozen=True) +class ProfileSpec: + """A named callable profile: ordered checkpoint IDs Connect can request.""" + + name: str + checkpoint_ids: tuple[str, ...] + description: str + + # --------------------------------------------------------------------------- # The registry. Order here is for readability only; lookups go through # resolve(). Keep this list aligned with the master-checklist registry/mint @@ -630,6 +639,125 @@ class ResolvedPlan: REGISTRY: dict[str, CheckpointSpec] = {spec.key: spec for spec in _SPECS} +_PROFILE_DEFINITIONS: tuple[ProfileSpec, ...] = ( + ProfileSpec( + name="workday-da:setup-readiness", + description="Foundation readiness before Workday Connect setup.", + checkpoint_ids=( + "ENV-001", + "ENV-002", + "ENV-CAPACITY-001", + "DA-AGENT-001", + "DA-CONTENT-001", + "ESS-SOLN-001", + "WD-PKG-001", + ), + ), + ProfileSpec( + name="workday-da:dataverse-ready", + description="Dataverse and bridge-package readiness for Workday DA.", + checkpoint_ids=( + "ENV-001", + "ENV-002", + "ENV-009", + "ESS-SOLN-001", + "WD-PKG-001", + "WD-FLOW", + "DV-CONN-001", + ), + ), + ProfileSpec( + name="workday-da:external-prerequisites", + description="External prerequisites owned by Entra and Workday admins.", + checkpoint_ids=( + "WD-ENTRA-SCOPE-001", + "WD-ENTRA-CONSENT-001", + "WD-ASSIGN-001", + "WD-ENTRA-NAMEID-001", + "WD-ENTRA-SIGNOPT-001", + "WD-API-CLIENT-001", + "WD-TENANT-001", + "WD-NET-001", + ), + ), + ProfileSpec( + name="workday-da:post-runtime", + description="Post-runtime Workday read-path validation.", + checkpoint_ids=("WD-RUN-001",), + ), + ProfileSpec( + name="workday-da:post-connection", + description="Connection-reference and Workday endpoint validation.", + checkpoint_ids=( + "WD-PKG-001", + "WD-CONN-012", + "WD-CONN-AUTH-001", + "WD-CONN-013", + "DV-CONN-001", + "WD-REST-001", + "WD-REST-002", + ), + ), + ProfileSpec( + name="workday-da:post-agent-wiring", + description="Topic and agent wiring validation after connection setup.", + checkpoint_ids=( + "DA-CONN", + "TOPIC-TRIGGER", + "TOPIC-INTEGRATION", + ), + ), + ProfileSpec( + name="workday-da:final", + description="Full DA Workday Connect final readiness profile.", + checkpoint_ids=( + "ENV-001", + "ENV-002", + "ENV-CAPACITY-001", + "DA-AGENT-001", + "DA-CONTENT-001", + "DA-CONN", + "ESS-SOLN-001", + "WD-PKG-001", + "WD-ENTRA-SCOPE-001", + "WD-ENTRA-CONSENT-001", + "WD-ASSIGN-001", + "WD-ENTRA-NAMEID-001", + "WD-ENTRA-SIGNOPT-001", + "WD-API-CLIENT-001", + "WD-TENANT-001", + "WD-CONN-012", + "WD-CONN-AUTH-001", + "WD-CONN-013", + "DV-CONN-001", + "WD-REST-001", + "WD-REST-002", + "WD-NET-001", + "WD-RUN-001", + "TOPIC-TRIGGER", + "TOPIC-INTEGRATION", + ), + ), + ProfileSpec( + name="workday-legacy:diagnostic", + description="Legacy/full Workday diagnostic profile retained separately.", + checkpoint_ids=( + "WD-PKG-001", + "WD-001", + "WD-CONN", + "WD-FLOW", + "WD-WF", + "WD-ENV", + ), + ), +) + +PROFILES: dict[str, ProfileSpec] = { + profile.name: profile for profile in _PROFILE_DEFINITIONS +} +_SHIPPED_REGISTRY_KEYS = frozenset(REGISTRY) + + # --------------------------------------------------------------------------- # Owned-prefix allow-list for the drift test (tests/flightcheck/ # test_registry_drift.py). This is the set of checkpoint-ID prefixes the @@ -800,6 +928,71 @@ def _order_index(label: str) -> int: ) +def resolve_profile(profile_name: str) -> Optional[ProfileSpec]: + """Return a callable validation profile by name.""" + return PROFILES.get(profile_name) + + +def list_profiles() -> list[ProfileSpec]: + """Return callable profiles sorted by stable profile name.""" + return sorted(PROFILES.values(), key=lambda profile: profile.name) + + +def profile_requirements(profile_name: str) -> ResolvedPlan: + """Resolve a profile to the union of its checkpoint execution needs.""" + profile = PROFILES.get(profile_name) + if profile is None: + raise RegistryError(f"Unknown profile {profile_name!r}.") + if not profile.checkpoint_ids: + raise RegistryError( + f"Profile {profile_name!r} declares no checkpoints." + ) + + clients: frozenset = frozenset() + requires_config = False + requires_dataverse_endpoint = False + seen_fns: set = set() + unique: list[tuple] = [] + + for checkpoint_id in profile.checkpoint_ids: + plan = transitive_requirements(checkpoint_id) + clients = clients | plan.clients + requires_config = requires_config or plan.requires_config + requires_dataverse_endpoint = ( + requires_dataverse_endpoint or plan.requires_dataverse_endpoint + ) + for label, fn in plan.ordered_fns: + if fn in seen_fns: + continue + seen_fns.add(fn) + unique.append((label, fn)) + + def _order_index(label: str) -> int: + try: + return CATEGORY_ORDER.index(label) + except ValueError: + return len(CATEGORY_ORDER) + + unique.sort(key=lambda pair: _order_index(pair[0])) + + return ResolvedPlan( + target=profile.name, + spec=_resolve_or_raise(profile.checkpoint_ids[0]), + clients=clients, + requires_config=requires_config, + requires_dataverse_endpoint=requires_dataverse_endpoint, + ordered_fns=unique, + ) + + +def profile_matches(profile_name: str, emitted_id: str) -> bool: + """True when an emitted checkpoint ID belongs to a profile member.""" + profile = PROFILES.get(profile_name) + if profile is None: + return False + return any(matches(checkpoint_id, emitted_id) for checkpoint_id in profile.checkpoint_ids) + + def matches(target: str, emitted_id: str) -> bool: """True if a runner-emitted checkpoint ID belongs to the requested target. @@ -856,6 +1049,16 @@ def validate_registry() -> None: f"checkpoint or family." ) + if frozenset(REGISTRY) == _SHIPPED_REGISTRY_KEYS: + for profile in PROFILES.values(): + for checkpoint_id in profile.checkpoint_ids: + if resolve(checkpoint_id) is None: + raise RegistryError( + f"Profile {profile.name!r} declares checkpoint " + f"{checkpoint_id!r}, which does not resolve to any " + f"registered checkpoint or family." + ) + # (2) The prereq graph (keyed by resolved spec key) must be acyclic. WHITE, GREY, BLACK = 0, 1, 2 color: dict[str, int] = {key: WHITE for key in REGISTRY} diff --git a/solutions/ess-maker-skills/scripts/flightcheck/runner.py b/solutions/ess-maker-skills/scripts/flightcheck/runner.py index cde97429e..b02d56257 100644 --- a/solutions/ess-maker-skills/scripts/flightcheck/runner.py +++ b/solutions/ess-maker-skills/scripts/flightcheck/runner.py @@ -13,12 +13,18 @@ import traceback from dataclasses import dataclass, field, asdict from enum import Enum -from typing import Callable +from typing import Any, Callable + + +FLIGHTCHECK_RESULT_SCHEMA_VERSION = "flightcheck.result.v1" class Status(str, Enum): PASSED = "Passed" FAILED = "Failed" + # BLOCKED is distinct from SKIPPED because an unavailable essential + # platform capability is a release gate, not a benign non-applicable row. + BLOCKED = "Blocked" WARNING = "Warning" NOT_CONFIGURED = "NotConfigured" SKIPPED = "Skipped" @@ -72,12 +78,28 @@ class CheckResult: remediation: str = "" # How to fix doc_link: str = "" # Microsoft Learn URL doc_label: str = "" # Link text for doc_link; falls back to "Docs" + severity: str = "" # Stable contract severity consumed by Connect + automation_type: str = "" # automated / manual / active_probe / passive + remediation_id: str = "" # Stable remediation contract identifier + evidence: dict[str, Any] = field(default_factory=dict) # Non-secret evidence # roles — the persona(s) who own the next step (fix or manual # validation). Every production check sets this; defaults to empty # so the runner's ERROR fallback and unit-test constructions still # build. Values are Role enum strings. roles: list[str] = field(default_factory=list) + def __post_init__(self) -> None: + if not self.severity: + self.severity = _default_severity(self.status, self.priority) + if not self.automation_type: + self.automation_type = ( + "manual" if self.status == Status.MANUAL.value else "automated" + ) + if not self.remediation_id: + self.remediation_id = self.checkpoint_id + if not self.evidence and self.result: + self.evidence = {"summary": self.result} + @dataclass class CategorySummary: @@ -85,6 +107,7 @@ class CategorySummary: total: int = 0 passed: int = 0 failed: int = 0 + blocked: int = 0 warnings: int = 0 not_configured: int = 0 skipped: int = 0 @@ -102,12 +125,59 @@ class RunResult: total: int = 0 passed: int = 0 failed: int = 0 + blocked: int = 0 warnings: int = 0 not_configured: int = 0 manual: int = 0 skipped: int = 0 errors: int = 0 overall: str = "" # READY / READY_WITH_WARNINGS / NOT_READY + profile: str = "" + profile_checkpoints: list[str] = field(default_factory=list) + validation_context: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ValidationContext: + """Non-secret context Connect passes to profile runs. + + Realm is required so Connect cannot accidentally treat a Dev validation + result as Test/Prod readiness. The remaining fields are identifiers only; + do not put tokens, credentials, employee IDs, or user data here. + """ + + realm: str + environment_id: str = "" + environment_url: str = "" + agent_schema_name: str = "" + agent_id: str = "" + tenant_id: str = "" + + def __post_init__(self) -> None: + if not isinstance(self.realm, str) or not self.realm.strip(): + raise ValueError("ValidationContext.realm is required.") + + def to_dict(self) -> dict[str, str]: + return { + "realm": self.realm.strip(), + "environmentId": self.environment_id.strip(), + "environmentUrl": self.environment_url.strip(), + "agentSchemaName": self.agent_schema_name.strip(), + "agentId": self.agent_id.strip(), + "tenantId": self.tenant_id.strip(), + } + + +def _default_severity(status: str, priority: str) -> str: + if status == Status.BLOCKED.value: + return "blocking" + if status in (Status.FAILED.value, Status.ERROR.value): + return str(priority or Priority.HIGH.value).lower() + if status == Status.WARNING.value: + return "warning" + if status in (Status.MANUAL.value, Status.NOT_CONFIGURED.value): + return "manual" + return "info" class FlightCheckRunner: @@ -185,6 +255,8 @@ def run(self) -> RunResult: s.passed += 1 elif r.status == Status.FAILED.value: s.failed += 1 + elif r.status == Status.BLOCKED.value: + s.blocked += 1 elif r.status == Status.WARNING.value: s.warnings += 1 elif r.status == Status.NOT_CONFIGURED.value: @@ -197,6 +269,7 @@ def run(self) -> RunResult: s.manual += 1 total_failed = sum(c.failed for c in cat_map.values()) + total_blocked = sum(c.blocked for c in cat_map.values()) total_warnings = sum(c.warnings for c in cat_map.values()) total_passed = sum(c.passed for c in cat_map.values()) # Tallied here so the verdict logic can consult errors. Errors @@ -209,9 +282,14 @@ def run(self) -> RunResult: # contradiction the prioritized report is meant to eliminate. total_errors = sum(c.errors for c in cat_map.values()) - if total_failed == 0 and total_errors == 0 and total_warnings == 0: + if ( + total_failed == 0 + and total_blocked == 0 + and total_errors == 0 + and total_warnings == 0 + ): overall = "READY" - elif total_failed == 0 and total_errors == 0: + elif total_failed == 0 and total_blocked == 0 and total_errors == 0: overall = "READY_WITH_WARNINGS" else: overall = "NOT_READY" @@ -225,6 +303,7 @@ def run(self) -> RunResult: total=len(self.results), passed=total_passed, failed=total_failed, + blocked=total_blocked, warnings=total_warnings, not_configured=sum(c.not_configured for c in cat_map.values()), manual=sum(c.manual for c in cat_map.values()), @@ -242,13 +321,18 @@ def save_results(run_result: RunResult, output_dir: str = "workspace/flightcheck # Write results.json results_path = os.path.join(output_dir, "results.json") data = { + "schema_version": FLIGHTCHECK_RESULT_SCHEMA_VERSION, "scope": run_result.scope, + "profile": run_result.profile, + "profile_checkpoints": run_result.profile_checkpoints, + "validation_context": run_result.validation_context, "started": run_result.started, "duration_secs": run_result.duration_secs, "overall": run_result.overall, "total": run_result.total, "passed": run_result.passed, "failed": run_result.failed, + "blocked": run_result.blocked, "warnings": run_result.warnings, "not_configured": run_result.not_configured, "manual": run_result.manual, @@ -256,6 +340,7 @@ def save_results(run_result: RunResult, output_dir: str = "workspace/flightcheck "errors": run_result.errors, "categories": [asdict(c) for c in run_result.categories], "results": [asdict(r) for r in run_result.results], + "contract": versioned_run_result_to_dict(run_result), } with open(results_path, "w", encoding="utf-8") as f: json.dump(data, f, indent=2) @@ -286,9 +371,11 @@ def save_results(run_result: RunResult, output_dir: str = "workspace/flightcheck # ------------------------------------------------------------------ # Results sort into one of three rendered sections, top to bottom: # -# 1. ACTION_REQUIRED — Failed, Error. These are checks that did -# not pass and the kit is confident the operator must act. -# The blocking signal — fix-this-now items only. +# 1. ACTION_REQUIRED — Failed, Blocked, Error. These are checks +# that did not pass and the kit is confident the operator must +# act. Blocked means an essential platform capability was +# unavailable; keeping it separate from Skipped prevents a +# release gate from being rendered as a success-shaped no-op. # # 2. MANUAL_VERIFICATION — Warning, Manual, NotConfigured. The # kit cannot make a yes/no judgement, or surfaced a soft @@ -298,11 +385,9 @@ def save_results(run_result: RunResult, output_dir: str = "workspace/flightcheck # verification path is the operator's, not the kit's. NotConfigured # means the kit had no creds/visibility to evaluate. # -# 3. PASSED — Passed, Skipped. Skipped is grouped with Passed -# because the kit chose not to run the check (e.g. it didn't -# apply to this scope, or a precondition wasn't met); from the -# operator's triage perspective the row needs no action and -# should sit alongside the proof-of-work passes. +# 3. PASSED — Passed, Skipped. Skipped is grouped with Passed only +# for benign non-applicable rows. Essential unavailable capability +# rows MUST use Blocked instead, so they land in Action required. # # Within each bucket, results are sorted by: # - priority (Critical > High > Medium > Low > unknown last) @@ -321,6 +406,7 @@ def save_results(run_result: RunResult, output_dir: str = "workspace/flightcheck # values (which is what CheckResult.status carries). _STATUS_TO_BUCKET = { Status.FAILED.value: BUCKET_ACTION, + Status.BLOCKED.value: BUCKET_ACTION, Status.ERROR.value: BUCKET_ACTION, Status.WARNING.value: BUCKET_MANUAL, Status.MANUAL.value: BUCKET_MANUAL, @@ -332,9 +418,10 @@ def save_results(run_result: RunResult, output_dir: str = "workspace/flightcheck # Within-bucket status sort order — lower index = surfaced first. # Worst news in each bucket goes to the top. _BUCKET_STATUS_ORDER = { - # ACTION_REQUIRED — Failed first, then Error. + # ACTION_REQUIRED — Failed first, then Blocked, then Error. Status.FAILED.value: 0, - Status.ERROR.value: 1, + Status.BLOCKED.value: 1, + Status.ERROR.value: 2, # MANUAL_VERIFICATION — Warning first because it carries an # observed finding (vs Manual/NotConfigured, which are "we # didn't / couldn't evaluate"). @@ -387,6 +474,56 @@ def bucket_results( return buckets +def check_result_contract_dict(result: CheckResult) -> dict[str, Any]: + """Versioned Connect-facing row shape with stable camelCase keys.""" + return { + "checkpointId": result.checkpoint_id, + "category": result.category, + "status": result.status, + "severity": result.severity, + "automationType": result.automation_type, + "remediationId": result.remediation_id, + "description": result.description, + "result": result.result, + "remediation": result.remediation, + "docLink": result.doc_link, + "docLabel": result.doc_label, + "roles": list(result.roles), + "evidence": dict(result.evidence), + } + + +def versioned_run_result_to_dict(run_result: RunResult) -> dict[str, Any]: + """Return the versioned JSON contract consumed by Connect. + + The legacy snake_case keys remain in ``results.json`` for existing report + consumers; this nested contract gives Connect stable camelCase fields and + a version string to negotiate future additive changes. + """ + return { + "schemaVersion": FLIGHTCHECK_RESULT_SCHEMA_VERSION, + "profile": run_result.profile, + "profileCheckpoints": list(run_result.profile_checkpoints), + "validationContext": dict(run_result.validation_context), + "overall": run_result.overall, + "counts": { + "total": run_result.total, + "passed": run_result.passed, + "failed": run_result.failed, + "blocked": run_result.blocked, + "warnings": run_result.warnings, + "notConfigured": run_result.not_configured, + "manual": run_result.manual, + "skipped": run_result.skipped, + "errors": run_result.errors, + }, + "results": [ + check_result_contract_dict(result) + for result in run_result.results + ], + } + + def _generate_html_report(r: RunResult) -> str: """Generate the category-grouped HTML report. @@ -444,7 +581,7 @@ def _verdict_text(r: RunResult) -> tuple[str, str, str, str]: operators at the right section is the whole reason the verdict has a subline. """ - failing = r.failed + r.errors + failing = r.failed + r.blocked + r.errors manual_count = r.warnings + r.manual + r.not_configured if r.overall == "READY": @@ -475,20 +612,23 @@ def _verdict_text(r: RunResult) -> tuple[str, str, str, str]: ) # NOT_READY (or any unrecognized overall) — treat as a blocker. - # Headline counts failures/errors as the truly blocking items; + # Headline counts failures/blocked/errors as the truly blocking items; # warnings (now in the manual section) are mentioned in the # subline so the operator knows their scale without thinking # they're additional blockers. issue_word = "issue" if failing == 1 else "issues" + blocked_text = ( + "failing/blocked/errored" if r.blocked else "failing/errored" + ) if r.warnings: sub = ( - f"{failing} failing/errored check(s) need action; " + f"{failing} {blocked_text} check(s) need action; " f"{r.warnings} warning(s) need manual verification. " "Start with \u201cAction required\u201d below." ) else: sub = ( - f"{failing} failing/errored check(s) need action. " + f"{failing} {blocked_text} check(s) need action. " "See \u201cAction required\u201d below." ) return ( @@ -1301,6 +1441,7 @@ def _render_synopsis( stats = [ ("pass", "green", r.passed, "Passed"), ("fail", "red", r.failed, "Failed"), + ("fail", "red", r.blocked, "Blocked"), ("warn", "amber", r.warnings, "Warning"), ("manual", "gray", r.manual, "Manual"), ("na", "gray", r.not_configured, "Not configured"), diff --git a/tests/flightcheck/contract/fixtures/workday_da_connections.json b/tests/flightcheck/contract/fixtures/workday_da_connections.json new file mode 100644 index 000000000..3470f669f --- /dev/null +++ b/tests/flightcheck/contract/fixtures/workday_da_connections.json @@ -0,0 +1,99 @@ +{ + "absent": { + "connectionReferenceChanges": [] + }, + "partial": { + "connectionReferenceChanges": [ + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "gptagent_mockemployeeselfservice.shared_workdaysoap", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": null + } + } + ] + }, + "legacy": { + "connectionReferenceChanges": [ + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "new_sharedworkdaysoap_ff0df", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": "mock-legacy-obo" + } + }, + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "new_sharedworkdaysoap_generic", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": "mock-legacy-generic-user" + } + }, + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "new_sharedworkdaysoap_context", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": "mock-legacy-context-generic-user" + } + } + ] + }, + "healthy": { + "connectionReferenceChanges": [ + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "new_sharedworkdaysoap_ff0df", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": "mock-healthy-workday", + "sharedConnectionParameters": "{\"values\":{\"restBaseUri\":{\"value\":\"https://wd.example.com/ccx/api\"},\"tenantName\":{\"value\":\"mocktenant\"},\"token:ResourceUri\":{\"value\":\"https://wd.example.com\"},\"token:WorkdayTokenUri\":{\"value\":\"https://wd.example.com/ccx/oauth2/mocktenant/token\"},\"token:WorkdayClientId\":{\"value\":\"mock-client-id\"}}}" + } + } + ] + }, + "degraded": { + "connectionReferenceChanges": [ + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "new_sharedworkdaysoap_ff0df", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": "mock-degraded-workday", + "sharedConnectionParameters": "{\"values\":{}}" + } + } + ] + }, + "invoker": { + "connectionReferenceChanges": [ + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "new_sharedworkdaysoap_ff0df", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": "mock-invoker-workday", + "authenticationType": "OAuthUser", + "sharedConnectionParameters": "{\"values\":{\"restBaseUri\":{\"value\":\"https://wd.example.com/ccx/api\"},\"tenantName\":{\"value\":\"mocktenant\"}}}" + } + } + ] + }, + "embedded": { + "connectionReferenceChanges": [ + { + "changeType": "Insert", + "connectionReference": { + "connectionReferenceLogicalName": "new_sharedworkdaysoap_ff0df", + "connectorId": "/providers/Microsoft.PowerApps/apis/shared_workdaysoap", + "connectionId": "mock-embedded-workday", + "authenticationType": "Raw", + "sharedConnectionParameters": "{\"values\":{\"restBaseUri\":{\"value\":\"https://wd.example.com/ccx/api\"},\"tenantName\":{\"value\":\"mocktenant\"}}}" + } + } + ] + } +} diff --git a/tests/flightcheck/contract/test_workday_da_profiles.py b/tests/flightcheck/contract/test_workday_da_profiles.py new file mode 100644 index 000000000..75307cc5d --- /dev/null +++ b/tests/flightcheck/contract/test_workday_da_profiles.py @@ -0,0 +1,197 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Contract tests for Workday DA validation profiles and Connect JSON output. + +Pure-logic only: no external API is called. The connection-state fixtures are +non-secret minimalBots ``connectionReferenceChanges`` shapes that mirror the +validated AgentBuilder components contract used by ``tests.mocks``. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from flightcheck import registry +from flightcheck.runner import ( + BUCKET_ACTION, + BUCKET_PASSED, + CheckResult, + FlightCheckRunner, + FLIGHTCHECK_RESULT_SCHEMA_VERSION, + Priority, + Role, + Status, + ValidationContext, + bucket_results, + versioned_run_result_to_dict, +) + + +FIXTURE_PATH = ( + Path(__file__).with_name("fixtures") / "workday_da_connections.json" +) + +EXPECTED_PROFILES = { + "workday-da:setup-readiness", + "workday-da:dataverse-ready", + "workday-da:external-prerequisites", + "workday-da:post-runtime", + "workday-da:post-connection", + "workday-da:post-agent-wiring", + "workday-da:final", + "workday-legacy:diagnostic", +} + +EXPECTED_CONNECTION_STATES = { + "absent", + "partial", + "legacy", + "healthy", + "degraded", + "invoker", + "embedded", +} + + +def _result(status: str = Status.PASSED.value) -> CheckResult: + return CheckResult( + checkpoint_id="WD-CONTRACT-001", + category="Workday", + priority=Priority.HIGH.value, + status=status, + description="Contract result", + result="Observed non-secret test evidence.", + remediation="Fix the profile contract.", + roles=[Role.ESS_MAKER.value], + ) + + +def test_validation_context_requires_explicit_realm() -> None: + with pytest.raises(ValueError, match="realm"): + ValidationContext(realm="") + + +def test_versioned_json_contract_shape_contains_connect_fields() -> None: + runner = FlightCheckRunner(scope="profile:workday-da:setup-readiness") + runner.register("Workday", lambda _runner: [_result()]) + run_result = runner.run() + run_result.profile = "workday-da:setup-readiness" + run_result.profile_checkpoints = ["WD-CONTRACT-001"] + run_result.validation_context = ValidationContext( + realm="dev", + environment_id="00000000-0000-0000-0000-000000001111", + agent_schema_name="gptagent_mockemployeeselfservice", + ).to_dict() + + payload = versioned_run_result_to_dict(run_result) + + assert payload["schemaVersion"] == FLIGHTCHECK_RESULT_SCHEMA_VERSION + assert payload["profile"] == "workday-da:setup-readiness" + assert payload["validationContext"]["realm"] == "dev" + row = payload["results"][0] + for key in ( + "checkpointId", + "status", + "severity", + "automationType", + "remediationId", + "evidence", + ): + assert key in row + assert row["checkpointId"] == "WD-CONTRACT-001" + assert row["evidence"]["summary"] == "Observed non-secret test evidence." + + +def test_blocked_is_action_required_and_not_ready() -> None: + blocked = _result(Status.BLOCKED.value) + passed = _result(Status.PASSED.value) + + buckets = bucket_results([blocked, passed]) + + assert blocked in buckets[BUCKET_ACTION] + assert blocked not in buckets[BUCKET_PASSED] + + runner = FlightCheckRunner(scope="test") + runner.register("Workday", lambda _runner: [blocked]) + run_result = runner.run() + assert run_result.blocked == 1 + assert run_result.overall == "NOT_READY" + + +def test_profiles_are_registered_with_resolvable_checkpoint_members() -> None: + profiles = {profile.name: profile for profile in registry.list_profiles()} + + assert set(profiles) == EXPECTED_PROFILES + for profile in profiles.values(): + assert profile.checkpoint_ids + for checkpoint_id in profile.checkpoint_ids: + assert registry.resolve(checkpoint_id) is not None + + +def test_profile_wd_conn_013_resolves_to_real_workday_check() -> None: + # WD-CONN-013 (agent connection OBO parameter sharing) is a fully + # implemented, tested check in checks/workday.py, emitted by + # run_workday_checks. It must resolve to the real WD-CONN family / + # Workday category, NOT a placeholder stub. + profile = registry.resolve_profile("workday-da:post-connection") + + assert profile is not None + assert "WD-CONN-013" in profile.checkpoint_ids + spec = registry.resolve("WD-CONN-013") + assert spec is not None + assert spec.category_label == "Workday" + assert registry.profile_matches("workday-da:post-connection", "WD-CONN-013") + + +def test_profile_plan_runs_wd_conn_013_via_real_workday_category() -> None: + plan = registry.profile_requirements("workday-da:post-connection") + labels = [label for label, _fn in plan.ordered_fns] + + # No placeholder "Profile Stubs" category exists; WD-CONN-013 runs inside + # the real Workday category. + assert "Profile Stubs" not in labels + assert "Workday" in labels + + +def test_profile_requirements_rejects_empty_profile(monkeypatch) -> None: + # A profile with no checkpoint members must fail loudly instead of + # indexing checkpoint_ids[0] and raising an opaque IndexError. + empty = registry.ProfileSpec( + name="workday-da:empty-guard", + checkpoint_ids=(), + description="Intentionally empty profile for the guard test.", + ) + monkeypatch.setitem(registry.PROFILES, empty.name, empty) + + with pytest.raises(registry.RegistryError, match="no checkpoints"): + registry.profile_requirements(empty.name) + + +def test_connection_state_fixtures_cover_required_states() -> None: + fixtures = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + + assert set(fixtures) == EXPECTED_CONNECTION_STATES + for state, payload in fixtures.items(): + changes = payload.get("connectionReferenceChanges") + assert isinstance(changes, list), state + for change in changes: + ref = change["connectionReference"] + assert str(ref["connectorId"]).endswith("/apis/shared_workdaysoap") + assert "password" not in json.dumps(ref).lower() + assert "secret" not in json.dumps(ref).lower() + + +@pytest.mark.parametrize("state", sorted(EXPECTED_CONNECTION_STATES)) +def test_connection_state_fixture_has_expected_shape(state: str) -> None: + fixtures = json.loads(FIXTURE_PATH.read_text(encoding="utf-8")) + changes = fixtures[state]["connectionReferenceChanges"] + + if state == "absent": + assert changes == [] + else: + assert changes + assert all("connectionReference" in change for change in changes)