diff --git a/.github/workflows/skill-evals.yml b/.github/workflows/skill-evals.yml index f7966d9..0aadbb6 100644 --- a/.github/workflows/skill-evals.yml +++ b/.github/workflows/skill-evals.yml @@ -37,8 +37,9 @@ jobs: if [ "${{ github.event_name }}" = "pull_request" ]; then OUT=$(python -m evals.select --base "origin/${{ github.base_ref }}") else - # nightly / manual: run everything - OUT=$(python -m evals.select --changed evals/manifest.yaml) + # nightly (schedule) / manual (workflow_dispatch): full matrix — + # all skills (flyte-evals, incl. real tier) and the kind smoke. + OUT=$(python -m evals.select --all) fi echo "$OUT" python - "$OUT" <<'PY' >> "$GITHUB_OUTPUT" @@ -61,22 +62,22 @@ jobs: with: python-version: "3.12" - run: pip install flyte>=2.5.0 pyyaml requests - - name: Configure Flyte auth - run: echo "auth via UNION_API_KEY secret" - env: - UNION_API_KEY: ${{ secrets.UNION_API_KEY }} - name: Run evals on demo.hosted env: - UNION_API_KEY: ${{ secrets.UNION_API_KEY }} - GLM_API_KEY: ${{ secrets.GLM_API_KEY }} + # The flyte v2 SDK authenticates to demo.hosted via an API key read from + # FLYTE_API_KEY, sourced from the repo secret DEMO_HOSTED_API_KEY. (The + # GLM judge key is supplied to task pods out-of-band via the Flyte + # secret `glm-api-key`, not through the CI environment.) + FLYTE_API_KEY: ${{ secrets.DEMO_HOSTED_API_KEY }} TIERS: ${{ github.event_name == 'schedule' && '["static","trajectory","real"]' || '["static","trajectory"]' }} + # run_ci.py submits the workflow, waits for the run to reach a terminal + # phase, and exits non-zero unless it SUCCEEDED — unlike `flyte run`, + # which exits 0 as soon as the run is launched. The run itself fails when + # any scenario fails (see eval_wf.main), so this gates on eval results. run: | - flyte --config evals/config/flyte.yaml run --copy-style loaded_modules \ - evals/workflows/eval_wf.py main \ + python -m evals.workflows.run_ci \ --skills '${{ needs.select.outputs.skills }}' \ --tiers "$TIERS" - # The workflow attaches the HTML scorecard to the run report; the run exits - # non-zero if any scenario fails (enforced inside aggregate/report). # 3) Real kind-in-Docker smoke, only when a kind skill changed (privileged). kind-smoke: diff --git a/evals/harness/evaluate.py b/evals/harness/evaluate.py index 1b5d539..2a9bfae 100644 --- a/evals/harness/evaluate.py +++ b/evals/harness/evaluate.py @@ -33,6 +33,7 @@ class ArmResult: judge: JudgeResult | None = None exit_code: int = 0 error: str = "" + unavailable: bool = False # harness CLI not installed -> skip (not a failure) @property def checks_passed(self) -> bool: @@ -68,10 +69,57 @@ def lift(self) -> float | None: return None return round(t.score - c.score, 4) + @property + def skipped(self) -> bool: + """No usable harness ran this scenario (agent CLI not installed).""" + t = self.arms.get("treatment") + return bool(t and t.unavailable) + + @property + def errored(self) -> bool: + """The treatment harness ran but crashed — a broken eval, distinct from + a skill regression (and distinct from a merely-unavailable harness).""" + t = self.arms.get("treatment") + return bool(t and t.error and not t.unavailable) + + @property + def status(self) -> str: + """One of: skipped | error | scored.""" + if self.skipped: + return "skipped" + if self.errored: + return "error" + return "scored" + + @property + def score(self) -> float | None: + """Treatment rating in [0,1] — the tracked signal — or None if not scored.""" + if self.status != "scored": + return None + t = self.arms.get("treatment") + return t.score if t else None + + @property + def checks_ok(self) -> bool: + """Whether the treatment arm's deterministic checks passed. These are the + reliable, non-LLM signal CI gates on.""" + t = self.arms.get("treatment") + return bool(t and t.checks_passed) + + @property + def is_regression(self) -> bool: + """The CI hard-gate: a scored *static* scenario whose deterministic + SKILL.md lint failed. Static lint is the only non-stochastic signal — + trajectory/real arms run an LLM agent, so their failures lower the tracked + `score`/rating instead of hard-failing CI (which would flake per PR).""" + return self.status == "scored" and self.tier == "static" and not self.checks_ok + @property def passed(self) -> bool: + """Scored and the treatment arm fully passed (checks + judge). Retained + for the scorecard; gating uses is_regression / status instead.""" t = self.arms.get("treatment") - return bool(t and t.passed) + return self.status == "scored" and bool(t and t.passed) def to_dict(self) -> dict: return { @@ -79,6 +127,11 @@ def to_dict(self) -> dict: "skill": self.skill, "tier": self.tier, "harness": self.harness, + "status": self.status, + "score": self.score, + "checks_ok": self.checks_ok, + "is_regression": self.is_regression, + "skipped": self.skipped, "passed": self.passed, "lift": self.lift, "arms": { @@ -88,6 +141,7 @@ def to_dict(self) -> dict: "checks_passed": r.checks_passed, "exit_code": r.exit_code, "error": r.error, + "unavailable": r.unavailable, "judge": None if r.judge is None else { "score": r.judge.score, "passed": r.judge.passed, @@ -104,6 +158,18 @@ def to_dict(self) -> dict: } +def skipped_scenario(scenario: Scenario, harness: str) -> ScenarioResult: + """A ScenarioResult marked skipped because `harness` isn't installed. Built + without spawning work — the caller filters unavailable harnesses up front so + they still appear on the scorecard without burning a task per skip.""" + res = ScenarioResult(scenario.id, scenario.skill, scenario.tier, harness=harness) + for arm in scenario.arms(): + res.arms[arm] = ArmResult( + arm=arm, unavailable=True, error=f"{harness} CLI not available", + ) + return res + + def evaluate_static(scenario: Scenario) -> ScenarioResult: skill_dir = REPO_ROOT / "plugins" / "flyte" / "skills" / scenario.skill results = lint_skill(skill_dir) @@ -122,7 +188,11 @@ def evaluate_scenario(scenario: Scenario, harness: str, glm: GLMConfig) -> Scena for arm in scenario.arms(): if not runner.is_available(): - res.arms[arm] = ArmResult(arm=arm, error=f"{harness} CLI not available") + # Harness CLI isn't installed — record as unavailable so the scenario + # is SKIPPED (excluded from rating and gating), not counted as a fail. + res.arms[arm] = ArmResult( + arm=arm, unavailable=True, error=f"{harness} CLI not available", + ) continue try: with make_sandbox(tier=scenario.tier, glm=glm) as sb: diff --git a/evals/kind_smoke/run.sh b/evals/kind_smoke/run.sh index 443ee14..8121011 100755 --- a/evals/kind_smoke/run.sh +++ b/evals/kind_smoke/run.sh @@ -38,25 +38,140 @@ nodes: EOF echo "==> in-cluster minio + postgres (throwaway deps for the smoke test)" +# Plain pinned manifests with official upstream images instead of the Bitnami +# Helm charts: Bitnami deprecated its free image catalog (Aug 2025), so those +# charts now pull rolling ':latest' tags from a shrinking registry, which is +# slow/flaky on 2-CPU CI runners (minio was hitting 'context deadline +# exceeded'). These are throwaway deps — the skill itself uses external hosted +# Postgres + S3/R2 — so determinism matters more than fidelity here. The +# Postgres Service keeps the name 'pg-postgresql' the flyte-binary step wires to. kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - -helm repo add bitnami https://charts.bitnami.com/bitnami >/dev/null -helm repo update >/dev/null -helm upgrade --install pg bitnami/postgresql -n "$NS" \ - --set auth.postgresPassword=flyte --set auth.database=flyte --wait --timeout 5m -helm upgrade --install minio bitnami/minio -n "$NS" \ - --set auth.rootUser=minio --set auth.rootPassword=miniostorage --wait --timeout 5m +kubectl apply -n "$NS" -f - <<'EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pg + labels: { app: pg } +spec: + replicas: 1 + selector: { matchLabels: { app: pg } } + template: + metadata: + labels: { app: pg } + spec: + containers: + - name: postgres + image: postgres:16-alpine + env: + - { name: POSTGRES_PASSWORD, value: flyte } + - { name: POSTGRES_DB, value: flyte } + - { name: PGDATA, value: /var/lib/postgresql/data/pgdata } + ports: [{ containerPort: 5432 }] + readinessProbe: + exec: { command: ["pg_isready", "-U", "postgres"] } + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - { name: data, mountPath: /var/lib/postgresql/data } + volumes: + - { name: data, emptyDir: {} } +--- +apiVersion: v1 +kind: Service +metadata: + name: pg-postgresql +spec: + selector: { app: pg } + ports: [{ port: 5432, targetPort: 5432 }] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + labels: { app: minio } +spec: + replicas: 1 + selector: { matchLabels: { app: minio } } + template: + metadata: + labels: { app: minio } + spec: + containers: + - name: minio + image: minio/minio:RELEASE.2025-04-08T15-41-24Z + args: ["server", "/data", "--console-address", ":9001"] + env: + - { name: MINIO_ROOT_USER, value: minio } + - { name: MINIO_ROOT_PASSWORD, value: miniostorage } + ports: [{ containerPort: 9000 }, { containerPort: 9001 }] + readinessProbe: + httpGet: { path: /minio/health/ready, port: 9000 } + initialDelaySeconds: 5 + periodSeconds: 5 + volumeMounts: + - { name: data, mountPath: /data } + volumes: + - { name: data, emptyDir: {} } +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + selector: { app: minio } + ports: + - { name: api, port: 9000, targetPort: 9000 } + - { name: console, port: 9001, targetPort: 9001 } +EOF +kubectl rollout status -n "$NS" deploy/pg --timeout=5m +kubectl rollout status -n "$NS" deploy/minio --timeout=5m + +echo "==> create the object-store bucket in minio (the chart expects it to exist)" +# The flyte-binary chart does not create the metadata/userdata bucket — with a +# real S3/R2 backend the user pre-creates it, so mirror that for the throwaway +# minio here, otherwise the control plane can't reach its bucket. +kubectl run mc --rm -i --restart=Never -n "$NS" --image=minio/mc:latest --command -- \ + sh -c "mc alias set m http://minio.${NS}.svc.cluster.local:9000 minio miniostorage && mc mb -p m/flyte" echo "==> install flyte-binary (per deploy-flyte-kind flyte-binary step)" helm repo add flyteorg https://flyteorg.github.io/flyte >/dev/null -helm repo update >/dev/null -# NOTE: values wiring (db + storage endpoints) mirrors the skill; kept minimal here. +helm repo update flyteorg >/dev/null +# Values wiring mirrors the skill's values-local.yaml. flyte-binary is v2, whose +# schema nests db under configuration.database.postgres.* and storage creds under +# configuration.storage.providerConfig.s3.* — the flat v1 keys are silently +# ignored, which leaves the DB host at its 127.0.0.1 default and hangs the +# wait-for-db init container forever. Point db at the in-cluster pg and storage +# at the in-cluster minio. +cat > "${TMPDIR:-/tmp}/fb-values.yaml" < assert flyte-binary pod is present" +echo "==> assert flyte-binary pod is Ready" kubectl get pods -n "$NS" kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=flyte-binary \ -n "$NS" --timeout=5m diff --git a/evals/report.py b/evals/report.py index 026c590..4fc9b5a 100644 --- a/evals/report.py +++ b/evals/report.py @@ -14,17 +14,137 @@ import sys -def _cell(passed: bool) -> str: - return "✅" if passed else "❌" +def _status(r: dict) -> str: + # Back-compat: old result dicts (pre-status) only had `passed`. + return r.get("status") or ("scored" if r.get("passed") else "scored") + + +def _cell(r: dict) -> str: + """Status glyph for a result row. + ✅ scored & passed · 🟡 scored, checks ok but judge soft-fail · + ❌ regression (deterministic check failed) · ⏭ skipped · ⚠ harness error.""" + st = _status(r) + if st == "skipped": + return "⏭" + if st == "error": + return "⚠" + if r.get("is_regression"): + return "❌" + return "✅" if r.get("passed") else "🟡" + + +def reason(result: dict) -> str: + """One-line, human-readable reason for a scenario's outcome — skip cause, + harness error, first failing deterministic check, or failed LLM judge.""" + st = _status(result) + arms = result.get("arms", {}) + if st == "skipped": + t = arms.get("treatment", {}) + return t.get("error") or "harness unavailable" + for arm, ar in arms.items(): + for ch in ar.get("checks", []): + if not ch["passed"]: + return f"[{arm}] check {ch['kind']}: {ch['detail']}" + if ar.get("error") and not ar.get("unavailable"): + return f"[{arm}] error: {ar['error']}" + judge = ar.get("judge") + if judge and not judge.get("passed", True): + rat = (judge.get("rationale") or "").strip().replace("\n", " ") + return f"[{arm}] judge {_fmt(judge.get('score'))}: {rat[:240]}" + return "no passing treatment arm" + + +def rating(results: list[dict]) -> dict: + """Aggregate the run into a tracked rating rather than a pass/fail tally. + + rating = mean treatment score over *scored* scenarios (0..1); skipped and + errored scenarios are excluded. `regressions` counts scored scenarios whose + deterministic checks failed — the reliable, non-LLM signal CI gates on.""" + scored = [r for r in results if _status(r) == "scored"] + skipped = [r for r in results if _status(r) == "skipped"] + errored = [r for r in results if _status(r) == "error"] + regressions = [r for r in scored if r.get("is_regression")] + scores = [r["score"] for r in scored if r.get("score") is not None] + lifts = [r["lift"] for r in scored if r.get("lift") is not None] + by_skill: dict[str, list[float]] = {} + for r in scored: + if r.get("score") is not None: + by_skill.setdefault(r["skill"], []).append(r["score"]) + return { + "total": len(results), + "scored": len(scored), + "skipped": len(skipped), + "errored": len(errored), + "regressions": len(regressions), + "rating": round(sum(scores) / len(scores), 4) if scores else None, + "mean_lift": round(sum(lifts) / len(lifts), 4) if lifts else None, + "per_skill_rating": { + k: round(sum(v) / len(v), 4) for k, v in sorted(by_skill.items()) + }, + } + + +def rating_line(results: list[dict]) -> str: + m = rating(results) + rr = "n/a" if m["rating"] is None else f"{m['rating']:.3f}" + lift = "n/a" if m["mean_lift"] is None else f"{m['mean_lift']:+.3f}" + return ( + f"rating: {rr} (mean treatment score over {m['scored']} scored) | " + f"lift: {lift} | {m['scored']} scored, {m['skipped']} skipped, " + f"{m['errored']} errored, {m['regressions']} regressions" + ) + + +def _skip_breakdown(skipped: list[dict]) -> str: + counts: dict[str, int] = {} + for r in skipped: + counts[r.get("harness") or "-"] = counts.get(r.get("harness") or "-", 0) + 1 + return ", ".join(f"{h}×{n}" for h, n in sorted(counts.items())) + + +def failure_report(results: list[dict]) -> str: + """Plaintext breakdown for task logs / CI output: the rating line, then + regressions (the gating signal) and errors in detail, and a one-line skip + summary — skips are expected when an agent CLI isn't installed.""" + lines = [rating_line(results)] + + def _fmt_rows(rows, glyph): + for r in sorted(rows, key=lambda r: (r["skill"], r["scenario_id"], r.get("harness") or "")): + lines.append( + f" {glyph} {r['scenario_id']} [{r.get('harness') or '-'}] " + f"({r['skill']}/{r['tier']}): {reason(r)}" + ) + + regressions = [r for r in results if r.get("is_regression")] + if regressions: + lines.append(f"\n{len(regressions)} regression(s) (static SKILL.md lint failed):") + _fmt_rows(regressions, "✗") + + errored = [r for r in results if _status(r) == "error"] + if errored: + lines.append(f"\n{len(errored)} errored (harness ran but crashed):") + _fmt_rows(errored, "⚠") + + skipped = [r for r in results if _status(r) == "skipped"] + if skipped: + lines.append(f"\n{len(skipped)} skipped (harness CLI unavailable): {_skip_breakdown(skipped)}") + + if not regressions and not errored: + lines.append("\nno regressions — deterministic checks pass on all scored scenarios") + return "\n".join(lines) def to_markdown(results: list[dict]) -> str: - total = len(results) - failed = [r for r in results if not r["passed"]] + m = rating(results) + rr = "n/a" if m["rating"] is None else f"{m['rating']:.3f}" + per_skill = " · ".join(f"{k} {v:.2f}" for k, v in m["per_skill_rating"].items()) lines = [ - f"### flyte-agent-plugin evals — {total - len(failed)}/{total} passing", + f"### flyte-agent-plugin evals — rating {rr}", + "", + f"{rating_line(results)}", "", - "| scenario | skill | harness | tier | pass | treat | ctrl | lift |", + *( [f"**Per-skill rating:** {per_skill}", ""] if per_skill else [] ), + "| scenario | skill | harness | tier | status | treat | ctrl | lift |", "|---|---|---|---|:--:|--:|--:|--:|", ] for r in results: @@ -34,17 +154,19 @@ def to_markdown(results: list[dict]) -> str: lift = "" if r.get("lift") is None else f"{r['lift']:+.2f}" lines.append( f"| {r['scenario_id']} | {r['skill']} | {r.get('harness') or '-'} | {r['tier']} | " - f"{_cell(r['passed'])} | {_fmt(t.get('score'))} | {_fmt(c.get('score'))} | {lift} |" + f"{_cell(r)} | {_fmt(t.get('score'))} | {_fmt(c.get('score'))} | {lift} |" ) - if failed: - lines += ["", "
Failure detail", ""] - for r in failed: + detail = [r for r in results if _status(r) in ("scored",) and r.get("is_regression")] + detail += [r for r in results if _status(r) == "error"] + if detail: + lines += ["", "
Regression / error detail", ""] + for r in detail: lines.append(f"- **{r['scenario_id']}** ({r.get('harness') or '-'}):") for arm, ar in r.get("arms", {}).items(): for ch in ar.get("checks", []): if not ch["passed"]: lines.append(f" - [{arm}] check `{ch['kind']}`: {ch['detail']}") - if ar.get("error"): + if ar.get("error") and not ar.get("unavailable"): lines.append(f" - [{arm}] error: {ar['error']}") lines.append("
") return "\n".join(lines) @@ -63,13 +185,14 @@ def to_html(results: list[dict]) -> str: f"{html.escape(r['skill'])}" f"{html.escape(r.get('harness') or '-')}" f"{html.escape(r['tier'])}" - f"{_cell(r['passed'])}" + f"{_cell(r)}" f"{_fmt(t.get('score'))}" f"{_fmt(c.get('score'))}" f"{lift}" "" ) - passed = sum(1 for r in results if r["passed"]) + m = rating(results) + rr = "n/a" if m["rating"] is None else f"{m['rating']:.3f}" return f""" flyte-agent-plugin evals -

flyte-agent-plugin evals — {passed}/{len(results)} passing

+

flyte-agent-plugin evals — rating {rr}

+

{html.escape(rating_line(results))}

- +{''.join(rows)}
scenarioskillharnesstierpasstreatmentcontrollift
statustreatmentcontrollift
""" @@ -105,8 +229,9 @@ def main(argv: list[str] | None = None) -> int: pathlib.Path(args.markdown).write_text(md) else: print(md) - failed = sum(1 for r in results if not r["passed"]) - return 1 if failed else 0 + # Non-zero only on regressions (scored scenarios with failing deterministic + # checks) — skipped harnesses and soft judge scores don't fail the report. + return 1 if rating(results)["regressions"] else 0 if __name__ == "__main__": diff --git a/evals/select.py b/evals/select.py index 0f08e75..77008ee 100644 --- a/evals/select.py +++ b/evals/select.py @@ -47,9 +47,9 @@ def is_shared_infra(path: str, manifest: Manifest) -> bool: return any(fnmatch.fnmatch(path, g) for g in manifest.shared_infra_globs) -def select(changed: list[str], manifest: Manifest, scenarios) -> dict: +def select(changed: list[str], manifest: Manifest, scenarios, force_all: bool = False) -> dict: by_skill = scenarios_by_skill(scenarios) - run_all = any(is_shared_infra(p, manifest) for p in changed) + run_all = force_all or any(is_shared_infra(p, manifest) for p in changed) if run_all: chosen_skills = sorted(by_skill) @@ -77,21 +77,26 @@ def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(prog="evals.select") ap.add_argument("--base", help="git ref to diff against (e.g. origin/main)") ap.add_argument("--changed", nargs="*", help="explicit changed file list") + ap.add_argument("--all", action="store_true", + help="run the full matrix (all skills, kind smoke, real tier) — " + "used for nightly/manual runs, independent of any diff") args = ap.parse_args(argv) repo_root = pathlib.Path(__file__).resolve().parents[1] manifest = Manifest.load() scenarios = load_scenarios() - if args.changed is not None: + if args.all: + changed = [] + elif args.changed is not None: changed = args.changed elif args.base: changed = changed_from_git(args.base, repo_root) else: - print("provide --base or --changed ", file=sys.stderr) + print("provide --base , --changed , or --all", file=sys.stderr) return 2 - print(json.dumps(select(changed, manifest, scenarios), indent=2)) + print(json.dumps(select(changed, manifest, scenarios, force_all=args.all), indent=2)) return 0 diff --git a/evals/workflows/eval_wf.py b/evals/workflows/eval_wf.py index 223097c..178019b 100644 --- a/evals/workflows/eval_wf.py +++ b/evals/workflows/eval_wf.py @@ -44,8 +44,23 @@ def eval_unit(unit: dict) -> dict: sc = scenarios[unit["scenario_id"]] glm = GLMConfig.from_env() if sc.tier == "static": - return evaluate_static(sc).to_dict() - return evaluate_scenario(sc, unit["harness"], glm).to_dict() + result = evaluate_static(sc).to_dict() + else: + result = evaluate_scenario(sc, unit["harness"], glm).to_dict() + + # Log this unit's verdict so each scenario is visible in its own map action's + # logs (not just in the aggregate). SCORE shows the tracked rating; SKIP means + # the agent CLI isn't installed; REGRESSION is a deterministic-check failure. + from evals.report import reason + st = result.get("status", "scored") + tag = {"skipped": "SKIP", "error": "ERROR"}.get( + st, "REGRESSION" if result.get("is_regression") else "SCORE") + score = result.get("score") + score_s = "" if score is None else f" score={score:.2f}" + suffix = "" if st == "scored" and not result.get("is_regression") else f": {reason(result)}" + print(f"{tag} {result['scenario_id']} [{result.get('harness') or '-'}] " + f"({result['skill']}/{result['tier']}){score_s}{suffix}", flush=True) + return result @env.task(report=True) @@ -53,13 +68,10 @@ def aggregate(results: list[dict]) -> dict: """Collect verdicts into a scorecard summary (also emits an HTML report).""" import flyte.report - from evals.report import to_html, to_markdown + from evals.report import rating, to_html, to_markdown - passed = sum(1 for r in results if r["passed"]) summary = { - "total": len(results), - "passed": passed, - "failed": len(results) - passed, + **rating(results), # total, scored, skipped, errored, regressions, rating, ... "markdown": to_markdown(results), "results": results, } @@ -76,12 +88,26 @@ def main(skills: list[str] | None = None, harnesses: list[str] | None = None, tiers: list[str] | None = None) -> dict: """Top-level workflow: build the matrix, fan out, aggregate.""" + from evals.harness.evaluate import skipped_scenario + from evals.harness.runners import get_runner from evals.harness.spec import load_scenarios tiers = tiers or ["static", "trajectory"] scenarios = load_scenarios() - + by_id = {s.id: s for s in scenarios} + + # main runs the same image as eval_unit, so harness availability here matches + # the workers. Filter unavailable harnesses up front: they become skipped + # results inline (still on the scorecard) instead of a task pod per skip. + def _available(h: str) -> bool: + try: + return get_runner(h).is_available() + except Exception: + return False + + avail_cache: dict[str, bool] = {} units: list[dict] = [] + skipped: list[dict] = [] for sc in scenarios: if sc.tier not in tiers: continue @@ -91,13 +117,51 @@ def main(skills: list[str] | None = None, units.append({"scenario_id": sc.id, "harness": None}) continue for h in (harnesses or list(sc.harnesses)): - units.append({"scenario_id": sc.id, "harness": h}) - - if not units: - return {"total": 0, "passed": 0, "failed": 0, "results": [], "markdown": "no units selected"} - - results = [r for r in flyte.map(eval_unit, units) if isinstance(r, dict)] - return aggregate(results) + ok = avail_cache.setdefault(h, _available(h)) + (units if ok else skipped).append({"scenario_id": sc.id, "harness": h}) + + if not units and not skipped: + return {"total": 0, "scored": 0, "skipped": 0, "errored": 0, + "regressions": 0, "rating": None, "results": [], "markdown": "no units selected"} + + results = [r for r in flyte.map(eval_unit, units) if isinstance(r, dict)] if units else [] + # Add skipped harnesses as synthetic results (no task spent) for visibility. + results += [skipped_scenario(by_id[u["scenario_id"]], u["harness"]).to_dict() + for u in skipped] + summary = aggregate(results) + + import os + + from evals.report import failure_report + + # Emit the rating + per-scenario breakdown to this action's logs, so the + # tracked score and any regressions are visible right next to the run (not + # only in the HTML scorecard on the report tab). + print(failure_report(summary["results"]), flush=True) + + # Gate the run — and therefore CI, which keys on the terminal phase — on + # REGRESSIONS only: static SKILL.md lint failures, the one deterministic, + # non-stochastic signal. A skipped harness (agent CLI not installed), a + # trajectory arm's low score, or a soft LLM-judge verdict never fails the run; + # those are tracked as the rating over time. An optional FLYTE_EVALS_MIN_RATING + # adds a floor for stricter runs. aggregate() already attached the scorecard. + regressions = [r for r in summary["results"] if r.get("is_regression")] + if regressions: + ids = ", ".join( + f"{r['scenario_id']}" + for r in sorted(regressions, key=lambda r: (r["skill"], r["scenario_id"])) + ) + raise RuntimeError( + f"{len(regressions)} static-lint regression(s): {ids}" + ) + + floor = os.environ.get("FLYTE_EVALS_MIN_RATING") + if floor and summary["rating"] is not None and summary["rating"] < float(floor): + raise RuntimeError( + f"eval rating {summary['rating']:.3f} is below the required " + f"minimum {float(floor):.3f}" + ) + return summary if __name__ == "__main__": diff --git a/evals/workflows/run_ci.py b/evals/workflows/run_ci.py new file mode 100644 index 0000000..886a5e9 --- /dev/null +++ b/evals/workflows/run_ci.py @@ -0,0 +1,101 @@ +"""CI driver: submit the eval workflow to demo.hosted, wait for it to finish, +and exit non-zero unless the run actually succeeded. + +`flyte run` (the CLI) is fire-and-forget — it exits 0 as soon as the run is +*launched*, regardless of whether the run later fails — so the CI job went green +even when the run failed on the backend. This driver submits the same workflow, +blocks on the terminal phase, and propagates the run's success/failure as the +process exit code so GitHub Actions reports it faithfully. + +Auth: the flyte v2 SDK reads the API key from the FLYTE_API_KEY env var (sourced +in CI from the DEMO_HOSTED_API_KEY repo secret); the key encodes the endpoint and +org, and project/domain/image-builder come from evals/config/flyte.yaml. + +Usage: + FLYTE_API_KEY=... python -m evals.workflows.run_ci \ + --skills '["flyte-sdk-author"]' --tiers '["static","trajectory"]' +""" + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import sys + +import yaml + +import flyte +from flyte.models import ActionPhase + +from evals.workflows.eval_wf import main + +# Repo root (…/evals/workflows/run_ci.py -> parents[2]); the copy-bundle base. +_REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def _config(path: str) -> tuple[str | None, str | None, str]: + """Read project/domain/image-builder from the flyte config (single source + of truth); the endpoint and org come from the API key itself.""" + with open(path) as f: + cfg = yaml.safe_load(f) or {} + task = cfg.get("task", {}) + builder = cfg.get("image", {}).get("builder", "remote") + return task.get("project"), task.get("domain"), builder + + +def run(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(prog="evals.workflows.run_ci") + ap.add_argument("--skills", required=True, help="JSON list of skills to run") + ap.add_argument("--tiers", required=True, help="JSON list of tiers to run") + ap.add_argument("--config", default="evals/config/flyte.yaml") + args = ap.parse_args(argv) + + if not os.getenv("FLYTE_API_KEY"): + print("::error::FLYTE_API_KEY is not set (expected from the " + "DEMO_HOSTED_API_KEY repo secret)", flush=True) + return 1 + + skills = json.loads(args.skills) or None + tiers = json.loads(args.tiers) + project, domain, builder = _config(args.config) + + # init_from_api_key reads FLYTE_API_KEY from the environment when api_key is + # None, and decodes the endpoint + org from it (headless — no browser flow). + # root_dir pins the copy-bundle base to the repo root so the whole project + # tree is shipped (see copy_style="all" below). + flyte.init_from_api_key( + project=project, domain=domain, image_builder=builder, root_dir=_REPO_ROOT, + ) + + # copy_style="all" ships the entire project tree, not just modules imported at + # submission time. The harness imports evals.harness.* lazily inside the tasks + # and reads data files off disk (evals/manifest.yaml, evals/scenarios/**, and + # plugins/flyte/skills/**), resolved relative to the repo root — none of which + # "loaded_modules" would copy, which is what caused ModuleNotFoundError. + run = flyte.with_runcontext(copy_style="all").run( + main, skills=skills, tiers=tiers, + ) + print(f"Submitted run: {run.name}\n {run.url}", flush=True) + + # Block until the run reaches a terminal phase, streaming status transitions. + run.wait() + phase = run.phase # a flyte.models.ActionPhase (str-enum); .name e.g. "SUCCEEDED" + print(f"Run terminal phase: {getattr(phase, 'name', phase)}", flush=True) + + if phase != ActionPhase.SUCCEEDED: + print(f"::error::Eval run {run.name} did not succeed " + f"({getattr(phase, 'name', phase)}) — {run.url}", flush=True) + # The per-scenario failure breakdown is printed to the run's `main` action + # logs (and the HTML scorecard is on its report tab); point the operator + # there rather than trying to re-fetch outputs from a failed run. + print(f"See the per-scenario breakdown in the run logs / report tab: {run.url}", + flush=True) + return 1 + print(f"Eval run {run.name} succeeded.", flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(run())