diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8947c29..b903ffa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,4 +127,10 @@ jobs: run: python -m agentforge.rag.ingest examples/banking-compliance/corpus - name: Run eval gate if: steps.guard.outputs.have_key == 'true' - run: python evals/run_evals.py --threshold 0.8 + run: python evals/run_evals.py --threshold 0.8 --out evals/results.json + - name: Upload eval report + if: steps.guard.outputs.have_key == 'true' && always() + uses: actions/upload-artifact@v4 + with: + name: eval-report + path: evals/results.json diff --git a/.gitignore b/.gitignore index d8b9c04..675006a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ Thumbs.db # Real k8s secret (only the .example template is committed) deploy/k8s/secret.yaml + +# Generated eval report (served by the API; produced by run_evals.py --out) +evals/results.json diff --git a/README.md b/README.md index c850ee7..1fd6cd2 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ A **banking compliance assistant** ships as the reference example — RAG over p - **Guardrails** — PII redaction, tool scoping, and policy enforcement via LangChain middleware. - **Pluggable observability** — LangSmith by default; Langfuse adapter for a fully self-hosted setup; Prometheus `/metrics`. - **Evals as a CI gate** — regression tests on retrieval quality and answer faithfulness that block bad deploys. -- **Admin console** — Angular UI for chat, trace inspection, eval scores, and the approval queue. +- **Admin console** — Angular UI: chat, the approval queue, knowledge base, eval scores, and live ops metrics. - **One-command local run** — `docker compose up` to a working agent + console. ## Architecture (high level) diff --git a/agentforge/api/evals.py b/agentforge/api/evals.py new file mode 100644 index 0000000..c895e71 --- /dev/null +++ b/agentforge/api/evals.py @@ -0,0 +1,22 @@ +"""Read the eval report that ``evals/run_evals.py --out`` produces. + +The API just serves the latest run; producing it is a CI/ops step (CI uploads it +as an artifact; locally you can run evals against the live stack). Absent or +malformed file -> ``None``, which the console renders as a "not run yet" state. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from agentforge.config import get_settings + + +def load_report() -> dict[str, Any] | None: + path = Path(get_settings().evals_results_path) + try: + return json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return None diff --git a/agentforge/api/main.py b/agentforge/api/main.py index 0ede8c4..2672243 100644 --- a/agentforge/api/main.py +++ b/agentforge/api/main.py @@ -4,6 +4,7 @@ - ``GET /health`` liveness probe. - ``GET /metrics`` Prometheus metrics (HTTP + domain counters). - ``GET /documents`` list ingested source documents + chunk counts. +- ``GET /evals`` latest eval report (or null if not run). - ``GET /approvals`` list runs paused awaiting human approval. - ``POST /chat`` run the agent; may pause for approval. - ``POST /chat/stream`` stream the answer token-by-token over SSE. @@ -34,6 +35,7 @@ ChatRequest, ChatResponse, DocumentSummaryItem, + EvalReport, PendingAction, PendingApprovalItem, ) @@ -140,6 +142,14 @@ def _record_domain_metrics(resp: ChatResponse) -> None: metrics.answers_total.labels(grounded).inc() +@app.get("/evals", response_model=EvalReport | None) +def evals() -> EvalReport | None: + from agentforge.api.evals import load_report + + report = load_report() + return EvalReport(**report) if report else None + + @app.get("/documents", response_model=list[DocumentSummaryItem]) def documents() -> list[DocumentSummaryItem]: from agentforge.rag.catalog import list_documents diff --git a/agentforge/api/schemas.py b/agentforge/api/schemas.py index 882885c..0e67878 100644 --- a/agentforge/api/schemas.py +++ b/agentforge/api/schemas.py @@ -45,3 +45,19 @@ class DocumentSummaryItem(BaseModel): source: str title: str chunks: int + + +class EvalCase(BaseModel): + id: str + question: str + passed: bool + detail: str + + +class EvalReport(BaseModel): + generated_at: str + threshold: float + passed: int + total: int + pass_rate: float + cases: list[EvalCase] = Field(default_factory=list) diff --git a/agentforge/config.py b/agentforge/config.py index fdebc82..190514b 100644 --- a/agentforge/config.py +++ b/agentforge/config.py @@ -81,6 +81,11 @@ class Settings(BaseSettings): auto_ingest: bool = Field(default=False) auto_ingest_corpus: str = Field(default="examples/banking-compliance/corpus") + # --- Evals ----------------------------------------------------------- + # JSON report written by evals/run_evals.py --out, served at GET /evals for + # the console's Eval view. Absent until a run produces it. + evals_results_path: str = Field(default="evals/results.json") + # --- API ------------------------------------------------------------- api_host: str = Field(default="0.0.0.0") api_port: int = Field(default=8000) diff --git a/apps/console/src/app/agent.service.ts b/apps/console/src/app/agent.service.ts index e400ae5..1a84b72 100644 --- a/apps/console/src/app/agent.service.ts +++ b/apps/console/src/app/agent.service.ts @@ -40,6 +40,22 @@ export interface DocumentSummaryItem { chunks: number; } +export interface EvalCase { + id: string; + question: string; + passed: boolean; + detail: string; +} + +export interface EvalReport { + generated_at: string; + threshold: number; + passed: number; + total: number; + pass_rate: number; + cases: EvalCase[]; +} + export type StreamEvent = | { type: 'thread'; threadId: string } | { type: 'token'; text: string } @@ -79,6 +95,11 @@ export class AgentService { return this.http.get(`${API_BASE}/documents`); } + /** Latest eval report, or null if evals haven't been run in this environment. */ + evals(): Observable { + return this.http.get(`${API_BASE}/evals`); + } + /** * Stream a chat response over SSE. Uses fetch (not EventSource) because the * endpoint is a POST. Emits typed events; unsubscribing aborts the request. diff --git a/apps/console/src/app/app.component.ts b/apps/console/src/app/app.component.ts index d1c8ee1..51d930d 100644 --- a/apps/console/src/app/app.component.ts +++ b/apps/console/src/app/app.component.ts @@ -15,6 +15,7 @@ import { RouterLink, RouterLinkActive, RouterOutlet } from '@angular/router'; Approvals Knowledge + Evals Operations diff --git a/apps/console/src/app/app.routes.ts b/apps/console/src/app/app.routes.ts index 6878fc0..863b682 100644 --- a/apps/console/src/app/app.routes.ts +++ b/apps/console/src/app/app.routes.ts @@ -1,6 +1,7 @@ import { Routes } from '@angular/router'; import { ApprovalsComponent } from './approvals.component'; import { ChatComponent } from './chat.component'; +import { EvalsComponent } from './evals.component'; import { KnowledgeComponent } from './knowledge.component'; import { MetricsComponent } from './metrics.component'; @@ -8,6 +9,7 @@ export const routes: Routes = [ { path: '', component: ChatComponent, title: 'AgentForge — Chat' }, { path: 'approvals', component: ApprovalsComponent, title: 'AgentForge — Approvals' }, { path: 'knowledge', component: KnowledgeComponent, title: 'AgentForge — Knowledge' }, + { path: 'evals', component: EvalsComponent, title: 'AgentForge — Evals' }, { path: 'ops', component: MetricsComponent, title: 'AgentForge — Operations' }, { path: '**', redirectTo: '' }, ]; diff --git a/apps/console/src/app/evals.component.ts b/apps/console/src/app/evals.component.ts new file mode 100644 index 0000000..b78b271 --- /dev/null +++ b/apps/console/src/app/evals.component.ts @@ -0,0 +1,154 @@ +import { Component, OnInit } from '@angular/core'; +import { CommonModule } from '@angular/common'; +import { AgentService, EvalReport } from './agent.service'; + +@Component({ + selector: 'app-evals', + standalone: true, + imports: [CommonModule], + template: ` +
+

Evals

+ + {{ report.generated_at | date: 'medium' }} + +
+ +
⚠️ {{ error }}
+ +
+ No eval report in this environment yet. Run the suite against a live stack: + 
python evals/run_evals.py --out evals/results.json
+ (CI runs this as a gate and uploads the report as a build artifact.) +
+ + +
+
+
{{ r.pass_rate * 100 | number: '1.0-0' }}%
+
Pass rate
+
threshold {{ r.threshold * 100 | number: '1.0-0' }}%
+
+
+
{{ r.passed }} / {{ r.total }}
+
Cases passing
+
+
+
{{ r.pass_rate >= r.threshold ? 'PASS' : 'FAIL' }}
+
Gate
+
+
+ + + + + + + + + + + + + + + + + + +
CaseQuestionDetail
{{ c.passed ? '✅' : '❌' }}{{ c.id }}{{ c.question }}{{ c.detail }}
+ + `, + styles: [ + ` + .ops-head { + display: flex; + align-items: baseline; + justify-content: space-between; + } + .muted { + font-size: 0.8rem; + color: #718096; + } + .card.empty { + color: #718096; + } + .tiles { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(160px, 1fr)); + gap: 12px; + margin: 12px 0; + } + .tile { + background: #fff; + border: 1px solid #e2e8f0; + border-radius: 12px; + padding: 16px; + } + .tile.bad { + border-color: #e53e3e; + background: #fff5f5; + } + .tile-value { + font-size: 1.8rem; + font-weight: 600; + } + .tile-label { + color: #2d3748; + margin-top: 4px; + } + .tile-hint { + font-size: 0.8rem; + color: #718096; + margin-top: 4px; + } + table.docs { + width: 100%; + border-collapse: collapse; + background: #fff; + border: 1px solid #e2e8f0; + border-radius: 12px; + overflow: hidden; + } + .docs th, + .docs td { + text-align: left; + padding: 10px 12px; + border-bottom: 1px solid #edf2f7; + font-size: 0.9rem; + vertical-align: top; + } + .docs th { + background: #f7fafc; + color: #4a5568; + font-weight: 600; + } + .docs td.fail { + color: #c53030; + } + .docs tr:last-child td { + border-bottom: 0; + } + `, + ], +}) +export class EvalsComponent implements OnInit { + report: EvalReport | null = null; + loaded = false; + error = ''; + + constructor(private agent: AgentService) {} + + ngOnInit(): void { + this.agent.evals().subscribe({ + next: (report) => { + this.report = report; + this.loaded = true; + }, + error: (e) => { + this.error = `Couldn't load evals: ${e?.message ?? e}`; + this.loaded = true; + }, + }); + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 1cfbaaa..373506a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,9 @@ services: condition: service_completed_successfully volumes: - ./examples:/app/examples:ro + # Writable so `docker compose exec api python evals/run_evals.py --out + # evals/results.json` can produce a report the Evals view then serves. + - ./evals:/app/evals console: build: ./apps/console diff --git a/docs/SMOKE_TEST.md b/docs/SMOKE_TEST.md index 3a490b2..9b1ee86 100644 --- a/docs/SMOKE_TEST.md +++ b/docs/SMOKE_TEST.md @@ -144,6 +144,9 @@ Open . an error rather than hanging. - [ ] Click **Knowledge** — a table lists the ingested sources (AML/KYC policy, product sheet) with their chunk counts. +- [ ] Click **Evals**. Before a run it shows a "not run yet" note; after + `docker compose exec api python evals/run_evals.py --out evals/results.json` + it shows the pass rate, gate PASS/FAIL, and a per-case table. - [ ] Click **Operations** — tiles show live counts (chat requests, grounded %, approvals, PII redactions, HTTP requests) that update as you use Chat. diff --git a/evals/run_evals.py b/evals/run_evals.py index 5d4d02b..a0816ec 100644 --- a/evals/run_evals.py +++ b/evals/run_evals.py @@ -19,11 +19,28 @@ import json import sys import uuid +from datetime import datetime, timezone from pathlib import Path REFUSAL_MARKERS = ("don't know", "do not know", "out of scope", "cannot", "no relevant") +def _write_report(path: Path, threshold: float, results: list[dict]) -> None: + """Persist a structured run so the console's Eval view can read it.""" + passed = sum(r["passed"] for r in results) + total = len(results) + report = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "threshold": threshold, + "passed": passed, + "total": total, + "pass_rate": passed / total if total else 0.0, + "cases": results, + } + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2), encoding="utf-8") + + def _load_dataset(path: Path) -> list[dict]: return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] @@ -56,6 +73,7 @@ def main() -> int: parser = argparse.ArgumentParser(description="Run AgentForge regression evals") parser.add_argument("--dataset", default=str(Path(__file__).parent / "dataset.jsonl")) parser.add_argument("--threshold", type=float, default=0.8) + parser.add_argument("--out", help="Write a JSON report here (for the console Eval view).") args = parser.parse_args() from agentforge.agents import get_compiled_graph @@ -63,14 +81,22 @@ def main() -> int: graph = get_compiled_graph() cases = _load_dataset(Path(args.dataset)) - passed = 0 + results: list[dict] = [] for case in cases: ok, detail = _evaluate_case(graph, case) - passed += ok + results.append( + {"id": case["id"], "question": case["question"], "passed": ok, "detail": detail} + ) print(f"[{'PASS' if ok else 'FAIL'}] {case['id']}: {detail}") + passed = sum(r["passed"] for r in results) rate = passed / len(cases) if cases else 0.0 print(f"\nPass rate: {passed}/{len(cases)} = {rate:.0%} (threshold {args.threshold:.0%})") + + if args.out: + _write_report(Path(args.out), args.threshold, results) + print(f"Wrote report to {args.out}") + return 0 if rate >= args.threshold else 1 diff --git a/tests/test_evals_report.py b/tests/test_evals_report.py new file mode 100644 index 0000000..2bd76ca --- /dev/null +++ b/tests/test_evals_report.py @@ -0,0 +1,65 @@ +"""Eval report writer (run_evals) + reader (/evals). No graph/LLM involved.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from fastapi.testclient import TestClient + +from agentforge.api.main import app + +EVALS_DIR = Path(__file__).resolve().parents[1] / "evals" + + +def test_write_report_roundtrip(tmp_path): + sys.path.insert(0, str(EVALS_DIR)) + import run_evals # noqa: E402 (added to path above) + + out = tmp_path / "results.json" + results = [ + {"id": "q1", "question": "when is EDD required?", "passed": True, "detail": "ok"}, + {"id": "q2", "question": "capital of France?", "passed": False, "detail": "did not refuse"}, + ] + run_evals._write_report(out, threshold=0.8, results=results) + + report = json.loads(out.read_text(encoding="utf-8")) + assert report["passed"] == 1 + assert report["total"] == 2 + assert report["pass_rate"] == 0.5 + assert report["threshold"] == 0.8 + assert len(report["cases"]) == 2 + assert "generated_at" in report + + +def test_evals_endpoint_null_when_absent(monkeypatch): + # Point the API at a path that doesn't exist -> 200 with null body. + from agentforge.api import evals as evals_module + + monkeypatch.setattr( + evals_module.get_settings(), "evals_results_path", "/nonexistent/results.json" + ) + resp = TestClient(app).get("/evals") + assert resp.status_code == 200 + assert resp.json() is None + + +def test_evals_endpoint_serves_report(monkeypatch, tmp_path): + report = { + "generated_at": "2026-06-08T00:00:00+00:00", + "threshold": 0.8, + "passed": 9, + "total": 10, + "pass_rate": 0.9, + "cases": [{"id": "q1", "question": "x", "passed": True, "detail": "ok"}], + } + path = tmp_path / "results.json" + path.write_text(json.dumps(report), encoding="utf-8") + + from agentforge.api import evals as evals_module + + monkeypatch.setattr(evals_module.get_settings(), "evals_results_path", str(path)) + body = TestClient(app).get("/evals").json() + assert body["pass_rate"] == 0.9 + assert body["cases"][0]["id"] == "q1"