flightcheck: add Workday DA profile + versioned result contract

solutions/ess-maker-skills/scripts/flightcheck/cli.py

@@ -41,6 +41,7 @@
 
 from flightcheck.runner import (
     FlightCheckRunner,
+    ValidationContext,
     save_results,
     Status,
     bucket_results,
@@ -710,6 +711,25 @@ def _is_native_no_dataverse(config: dict, env_url: str) -> bool:
     return str(active.get("releaseLine") or "").casefold() == "da"
 
 
+def _validation_context_from_args(
+    args,
+    config: dict,
+    env_url: str,
+    env_id: str | None,
+) -> ValidationContext:
+    active = _active_agent_config(config)
+    return ValidationContext(
+        realm=getattr(args, "validation_realm", None) or config.get("realm", ""),
+        environment_id=env_id or config.get("environmentId", ""),
+        environment_url=env_url or config.get("dataverseEndpoint", ""),
+        agent_schema_name=(
+            getattr(args, "agent_schema_name", None)
+            or active.get("schemaName", "")
+        ),
+        agent_id=active.get("botId", ""),
+    )
+
+
 def _resolve_environment_ring(
     config: dict,
     *,
@@ -1159,6 +1179,231 @@ def _run_single_checkpoint(args):
     sys.exit(1 if result.failed > 0 or result.errors > 0 else 0)
 
 
+def _run_profile(args):
+    """Run a named validation profile and emit the versioned result contract."""
+    from flightcheck import registry
+
+    profile_name = args.profile
+    profile = registry.resolve_profile(profile_name)
+    if profile is None:
+        print(f"ERROR: Unknown profile {profile_name!r}.")
+        print("Valid profiles:")
+        for item in registry.list_profiles():
+            print(f"  {item.name}")
+        sys.exit(2)
+
+    plan = registry.profile_requirements(profile_name)
+    needed = plan.clients
+
+    config = {}
+    config_path = os.path.join(".local", "config.json")
+    if os.path.exists(config_path):
+        with open(config_path, "r", encoding="utf-8") as f:
+            config = json.load(f)
+    elif plan.requires_config:
+        print("ERROR: .local/config.json not found. Run /setup first.")
+        sys.exit(1)
+
+    env_url = args.environment_url or config.get("dataverseEndpoint", "")
+    if plan.requires_dataverse_endpoint and not env_url:
+        print("ERROR: No dataverseEndpoint in .local/config.json.")
+        sys.exit(1)
+
+    quiet_auth = getattr(args, "quiet_auth", False)
+    if not quiet_auth:
+        print()
+        print("=" * 64)
+        print("  ESS FLIGHTCHECK — Validation Profile")
+        print("=" * 64)
+        print(f"  Profile:     {profile_name}")
+        if env_url:
+            print(f"  Environment: {env_url}")
+        print(f"  Clients:     {', '.join(sorted(needed)) or '(none)'}")
+        print("=" * 64)
+        print()
+
+    dv_token = None
+    tenant_id = None
+    graph = None
+    pp_admin = None
+    pva = None
+    powerplatform = None
+    agentbuilder = None
+    connectivity = None
+
+    if needed & {
+        registry.GRAPH,
+        registry.PP_ADMIN,
+        registry.PVA,
+        registry.DATAVERSE,
+        registry.POWERPLATFORM,
+    }:
+        from auth import discover_tenant
+        if env_url:
+            try:
+                tenant_id = discover_tenant(env_url)
+            except Exception as e:
+                print(f"  Tenant discovery: WARNING — {e}")
+                tenant_id = "organizations"
+        else:
+            tenant_id = "organizations"
+
+    if registry.DATAVERSE in needed and env_url:
+        from auth import authenticate
+        if not quiet_auth:
+            print("Authenticating to Dataverse...")
+        try:
+            dv_token = authenticate(env_url)
+            if not quiet_auth:
+                print("  Dataverse: OK")
+        except Exception as e:
+            print(f"  Dataverse: WARNING — {e}")
+            dv_token = None
+
+    if registry.GRAPH in needed:
+        if not quiet_auth:
+            print("Authenticating to Microsoft Graph...")
+        graph = GraphClient(tenant_id)
+        try:
+            graph.authenticate()
+            if not quiet_auth:
+                print("  Graph: OK")
+        except Exception as e:
+            print(f"  Graph: WARNING — {e}")
+            graph = None
+
+    if registry.PP_ADMIN in needed:
+        if not quiet_auth:
+            print("Authenticating to Power Platform Admin API...")
+        pp_admin = PPAdminClient(tenant_id)
+        try:
+            pp_admin.authenticate()
+            if not quiet_auth:
+                print("  Power Platform: OK")
+        except Exception as e:
+            print(f"  Power Platform: WARNING — {e}")
+            pp_admin = None
+
+    env_id = args.environment_id or config.get("environmentId") or None
+    if not env_id and registry.PP_ADMIN in needed and env_url:
+        env_id = derive_environment_id(env_url, dv_token, pp_admin=pp_admin)
+
+    if needed & {registry.AGENTBUILDER, registry.CONNECTIVITY}:
+        native_host = config.get("powerPlatformApiEndpoint", "")
+        if not native_host:
+            print(
+                "ERROR: No powerPlatformApiEndpoint in .local/config.json. "
+                "Run /setup again."
+            )
+            sys.exit(1)
+        try:
+            native_ring = ring_from_environment_host(native_host)
+            native_host = validate_environment_host(native_host, native_ring)
+        except ValueError as e:
+            print(f"ERROR: {e}")
+            sys.exit(1)
+        if not quiet_auth:
+            print("Authenticating to native AgentBuilder APIs...")
+        try:
+            native_token, native_tenant_id = authenticate_flightcheck(
+                native_ring,
+                include_connectivity=registry.CONNECTIVITY in needed,
+            )
+            tenant_id = native_tenant_id
+            agentbuilder = AgentBuilderClient(
+                native_host,
+                native_token,
+                ring=native_ring,
+                tenant_id=native_tenant_id,
+                api_version=config.get(
+                    "agentBuilderApiVersion", "2024-10-01"
+                ),
+            )
+            if registry.CONNECTIVITY in needed:
+                connectivity = ConnectivityClient(
+                    native_token,
+                    ring=native_ring,
+                    api_version=config.get(
+                        "agentBuilderApiVersion", "2024-10-01"
+                    ),
+                )
+            if not quiet_auth:
+                print("  Native AgentBuilder APIs: OK")
+        except Exception as e:
+            print(f"  Native AgentBuilder APIs: WARNING — {e}")
+            agentbuilder = None
+            connectivity = None
+
+    if registry.PVA in needed:
+        if not quiet_auth:
+            print("Authenticating to Copilot Studio (Island Gateway)...")
+        pva = PVAClient(tenant_id, env_url)
+        try:
+            pva.authenticate()
+            if not quiet_auth:
+                print("  Copilot Studio: OK")
+        except Exception as e:
+            print(f"  Copilot Studio: WARNING — {e}")
+            pva = None
+
+    if registry.POWERPLATFORM in needed:
+        if not quiet_auth:
+            print("Authenticating to Power Platform API (capacity allocation)...")
+        powerplatform = PowerPlatformClient(tenant_id)
+        try:
+            powerplatform.authenticate()
+            if not quiet_auth:
+                print("  Power Platform API: OK")
+        except Exception as e:
+            print(f"  Power Platform API: WARNING — {e}")
+            powerplatform = None
+
+    try:
+        validation_context = _validation_context_from_args(
+            args, config, env_url, env_id
+        )
+    except ValueError as e:
+        print(f"ERROR: {e}")
+        print("Pass --validation-realm dev|test|prod (or set \"realm\" in "
+              ".local/config.json) for profile runs.")
+        sys.exit(1)
+
+    runner = FlightCheckRunner(
+        scope=f"profile:{profile_name}",
+        target_matcher=lambda cid: registry.profile_matches(profile_name, cid),
+    )
+    runner.config = config
+    runner.env_url = env_url
+    runner.dv_token = dv_token
+    runner.env_id = env_id
+    runner.graph = graph
+    runner.pp_admin = pp_admin
+    runner.pva = pva
+    runner.powerplatform = powerplatform
+    runner.azure_arm = None
+    runner.agentbuilder = agentbuilder
+    runner.connectivity = connectivity
+
+    for label, fn in plan.ordered_fns:
+        runner.register(label, fn)
+
+    if not quiet_auth:
+        print("\nRunning profile...\n")
+    result = runner.run()
+    result.profile = profile_name
+    result.profile_checkpoints = list(profile.checkpoint_ids)
+    result.validation_context = validation_context.to_dict()
+
+    _print_prioritized_summary(result, verbose_manual=True)
+    save_results(result, args.output)
+
+    if not result.results:
+        print(f"\nNOTE: profile {profile_name} produced no result rows.")
+        sys.exit(1)
+
+    sys.exit(1 if result.failed > 0 or result.blocked > 0 or result.errors > 0 else 0)
+
+
 def main():
     # Force UTF-8 console output so summary glyphs (→, •) don't crash on
     # Windows cp1252 terminals. Without this, _print_prioritized_summary
@@ -1172,7 +1417,7 @@ def main():
     parser.add_argument(
         "--scope", default=None,
         choices=["full"] + list(SCOPE_MAP.keys()),
-        help="Validation scope (default: full). Mutually exclusive with --checkpoint.",
+        help="Validation scope (default: full). Mutually exclusive with --checkpoint/--profile.",
     )
     parser.add_argument(
         "--output", default="workspace/flightcheck",
@@ -1203,7 +1448,24 @@ def main():
         help="Run exactly one checkpoint (or a family, e.g. WD-FLOW-*) by ID and "
              "report only its result. Hydrates the checkpoint's declared "
              "prerequisites and initialises only the clients it needs. Mutually "
-             "exclusive with --scope.",
+             "exclusive with --scope/--profile.",
+    )
+    parser.add_argument(
+        "--profile",
+        help="Run a named validation profile, e.g. workday-da:setup-readiness. "
+             "Profile runs need a realm (--validation-realm, or a \"realm\" key "
+             "in .local/config.json) and emit the versioned Connect result "
+             "contract in results.json.",
+    )
+    parser.add_argument(
+        "--validation-realm",
+        choices=["dev", "test", "prod"],
+        help="Realm for a --profile validation context. Required with --profile "
+             "unless .local/config.json supplies a \"realm\" value.",
+    )
+    parser.add_argument(
+        "--agent-schema-name",
+        help="Agent schema name for a --profile validation context.",
     )
     parser.add_argument(
         "--connect-config",
@@ -1294,7 +1556,7 @@ def main():
         except ValueError as e:
             parser.error(f"invalid --agent-slug: {e}")
 
-    # --- Single-checkpoint mode (additive; leaves all --scope behavior intact) ---
+    # --- Targeted modes (additive; leave --scope behavior intact) ---
     if args.list_checkpoints:
         _print_checkpoint_list()
         sys.exit(0)
@@ -1307,12 +1569,19 @@ def main():
         sys.exit(0)
 
     if args.checkpoint:
-        if args.scope is not None:
-            print("ERROR: --checkpoint and --scope are mutually exclusive.")
+        if args.scope is not None or args.profile:
+            print("ERROR: --checkpoint is mutually exclusive with --scope/--profile.")
             sys.exit(2)
         _run_single_checkpoint(args)
         return  # _run_single_checkpoint always exits; defensive only.
 
+    if args.profile:
+        if args.scope is not None:
+            print("ERROR: --profile and --scope are mutually exclusive.")
+            sys.exit(2)
+        _run_profile(args)
+        return  # _run_profile always exits; defensive only.
+
     # Normal scope mode: --scope defaults to "full" when omitted. (Default is
     # None on the parser so checkpoint-mode can detect an explicit --scope.)
     if args.scope is None: