Phase 5: eval dashboard (GET /evals + console view)

.github/workflows/ci.yml

@@ -127,4 +127,10 @@ jobs:
         run: python -m agentforge.rag.ingest examples/banking-compliance/corpus
       - name: Run eval gate
         if: steps.guard.outputs.have_key == 'true'
-        run: python evals/run_evals.py --threshold 0.8
+        run: python evals/run_evals.py --threshold 0.8 --out evals/results.json
+      - name: Upload eval report
+        if: steps.guard.outputs.have_key == 'true' && always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-report
+          path: evals/results.json

.gitignore

@@ -15,3 +15,6 @@ Thumbs.db
 
 # Real k8s secret (only the .example template is committed)
 deploy/k8s/secret.yaml
+
+# Generated eval report (served by the API; produced by run_evals.py --out)
+evals/results.json

README.md

@@ -25,7 +25,7 @@ A **banking compliance assistant** ships as the reference example — RAG over p
 - **Guardrails** — PII redaction, tool scoping, and policy enforcement via LangChain middleware.
 - **Pluggable observability** — LangSmith by default; Langfuse adapter for a fully self-hosted setup; Prometheus `/metrics`.
 - **Evals as a CI gate** — regression tests on retrieval quality and answer faithfulness that block bad deploys.
-- **Admin console** — Angular UI for chat, trace inspection, eval scores, and the approval queue.
+- **Admin console** — Angular UI: chat, the approval queue, knowledge base, eval scores, and live ops metrics.
 - **One-command local run** — `docker compose up` to a working agent + console.
 
 ## Architecture (high level)

agentforge/api/evals.py

@@ -0,0 +1,22 @@
+"""Read the eval report that ``evals/run_evals.py --out`` produces.
+
+The API just serves the latest run; producing it is a CI/ops step (CI uploads it
+as an artifact; locally you can run evals against the live stack). Absent or
+malformed file -> ``None``, which the console renders as a "not run yet" state.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+from agentforge.config import get_settings
+
+
+def load_report() -> dict[str, Any] | None:
+    path = Path(get_settings().evals_results_path)
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except (FileNotFoundError, json.JSONDecodeError, OSError):
+        return None

agentforge/api/main.py

@@ -4,6 +4,7 @@
 - ``GET  /health``        liveness probe.
 - ``GET  /metrics``       Prometheus metrics (HTTP + domain counters).
 - ``GET  /documents``     list ingested source documents + chunk counts.
+- ``GET  /evals``         latest eval report (or null if not run).
 - ``GET  /approvals``     list runs paused awaiting human approval.
 - ``POST /chat``          run the agent; may pause for approval.
 - ``POST /chat/stream``   stream the answer token-by-token over SSE.
@@ -34,6 +35,7 @@
     ChatRequest,
     ChatResponse,
     DocumentSummaryItem,
+    EvalReport,
     PendingAction,
     PendingApprovalItem,
 )
@@ -140,6 +142,14 @@ def _record_domain_metrics(resp: ChatResponse) -> None:
         metrics.answers_total.labels(grounded).inc()
 
 
+@app.get("/evals", response_model=EvalReport | None)
+def evals() -> EvalReport | None:
+    from agentforge.api.evals import load_report
+
+    report = load_report()
+    return EvalReport(**report) if report else None
+
+
 @app.get("/documents", response_model=list[DocumentSummaryItem])
 def documents() -> list[DocumentSummaryItem]:
     from agentforge.rag.catalog import list_documents

agentforge/api/schemas.py

@@ -45,3 +45,19 @@ class DocumentSummaryItem(BaseModel):
     source: str
     title: str
     chunks: int
+
+
+class EvalCase(BaseModel):
+    id: str
+    question: str
+    passed: bool
+    detail: str
+
+
+class EvalReport(BaseModel):
+    generated_at: str
+    threshold: float
+    passed: int
+    total: int
+    pass_rate: float
+    cases: list[EvalCase] = Field(default_factory=list)

agentforge/config.py

@@ -81,6 +81,11 @@ class Settings(BaseSettings):
     auto_ingest: bool = Field(default=False)
     auto_ingest_corpus: str = Field(default="examples/banking-compliance/corpus")
 
+    # --- Evals -----------------------------------------------------------
+    # JSON report written by evals/run_evals.py --out, served at GET /evals for
+    # the console's Eval view. Absent until a run produces it.
+    evals_results_path: str = Field(default="evals/results.json")
+
     # --- API -------------------------------------------------------------
     api_host: str = Field(default="0.0.0.0")
     api_port: int = Field(default=8000)

apps/console/src/app/agent.service.ts

@@ -40,6 +40,22 @@ export interface DocumentSummaryItem {
   chunks: number;
 }
 
+export interface EvalCase {
+  id: string;
+  question: string;
+  passed: boolean;
+  detail: string;
+}
+
+export interface EvalReport {
+  generated_at: string;
+  threshold: number;
+  passed: number;
+  total: number;
+  pass_rate: number;
+  cases: EvalCase[];
+}
+
 export type StreamEvent =
   | { type: 'thread'; threadId: string }
   | { type: 'token'; text: string }
@@ -79,6 +95,11 @@ export class AgentService {
     return this.http.get<DocumentSummaryItem[]>(`${API_BASE}/documents`);
   }
 
+  /** Latest eval report, or null if evals haven't been run in this environment. */
+  evals(): Observable<EvalReport | null> {
+    return this.http.get<EvalReport | null>(`${API_BASE}/evals`);
+  }
+
   /**
    * Stream a chat response over SSE. Uses fetch (not EventSource) because the
    * endpoint is a POST. Emits typed events; unsubscribing aborts the request.

apps/console/src/app/app.component.ts

@@ -15,6 +15,7 @@ import { RouterLink, RouterLinkActive, RouterOutlet } from '@angular/router';
           </a>
           <a routerLink="/approvals" routerLinkActive="active">Approvals</a>
           <a routerLink="/knowledge" routerLinkActive="active">Knowledge</a>
+          <a routerLink="/evals" routerLinkActive="active">Evals</a>
           <a routerLink="/ops" routerLinkActive="active">Operations</a>
         </nav>
       </header>

apps/console/src/app/app.routes.ts

@@ -1,13 +1,15 @@
 import { Routes } from '@angular/router';
 import { ApprovalsComponent } from './approvals.component';
 import { ChatComponent } from './chat.component';
+import { EvalsComponent } from './evals.component';
 import { KnowledgeComponent } from './knowledge.component';
 import { MetricsComponent } from './metrics.component';
 
 export const routes: Routes = [
   { path: '', component: ChatComponent, title: 'AgentForge — Chat' },
   { path: 'approvals', component: ApprovalsComponent, title: 'AgentForge — Approvals' },
   { path: 'knowledge', component: KnowledgeComponent, title: 'AgentForge — Knowledge' },
+  { path: 'evals', component: EvalsComponent, title: 'AgentForge — Evals' },
   { path: 'ops', component: MetricsComponent, title: 'AgentForge — Operations' },
   { path: '**', redirectTo: '' },
 ];

apps/console/src/app/evals.component.ts

@@ -0,0 +1,154 @@
+import { Component, OnInit } from '@angular/core';
+import { CommonModule } from '@angular/common';
+import { AgentService, EvalReport } from './agent.service';
+
+@Component({
+  selector: 'app-evals',
+  standalone: true,
+  imports: [CommonModule],
+  template: `
+    <div class="ops-head">
+      <h2>Evals</h2>
+      <span class="muted" *ngIf="report">
+        {{ report.generated_at | date: 'medium' }}
+      </span>
+    </div>
+
+    <div class="pii" *ngIf="error">⚠️ {{ error }}</div>
+
+    <div class="card empty" *ngIf="loaded && !error && !report">
+      No eval report in this environment yet. Run the suite against a live stack:
+      <pre>python evals/run_evals.py --out evals/results.json</pre>
+      (CI runs this as a gate and uploads the report as a build artifact.)
+    </div>
+
+    <ng-container *ngIf="report as r">
+      <div class="tiles">
+        <div class="tile" [class.bad]="r.pass_rate < r.threshold">
+          <div class="tile-value">{{ r.pass_rate * 100 | number: '1.0-0' }}%</div>
+          <div class="tile-label">Pass rate</div>
+          <div class="tile-hint">threshold {{ r.threshold * 100 | number: '1.0-0' }}%</div>
+        </div>
+        <div class="tile">
+          <div class="tile-value">{{ r.passed }} / {{ r.total }}</div>
+          <div class="tile-label">Cases passing</div>
+        </div>
+        <div class="tile">
+          <div class="tile-value">{{ r.pass_rate >= r.threshold ? 'PASS' : 'FAIL' }}</div>
+          <div class="tile-label">Gate</div>
+        </div>
+      </div>
+
+      <table class="docs">
+        <thead>
+          <tr>
+            <th></th>
+            <th>Case</th>
+            <th>Question</th>
+            <th>Detail</th>
+          </tr>
+        </thead>
+        <tbody>
+          <tr *ngFor="let c of r.cases">
+            <td>{{ c.passed ? '✅' : '❌' }}</td>
+            <td><code>{{ c.id }}</code></td>
+            <td>{{ c.question }}</td>
+            <td [class.fail]="!c.passed">{{ c.detail }}</td>
+          </tr>
+        </tbody>
+      </table>
+    </ng-container>
+  `,
+  styles: [
+    `
+      .ops-head {
+        display: flex;
+        align-items: baseline;
+        justify-content: space-between;
+      }
+      .muted {
+        font-size: 0.8rem;
+        color: #718096;
+      }
+      .card.empty {
+        color: #718096;
+      }
+      .tiles {
+        display: grid;
+        grid-template-columns: repeat(auto-fill, minmax(160px, 1fr));
+        gap: 12px;
+        margin: 12px 0;
+      }
+      .tile {
+        background: #fff;
+        border: 1px solid #e2e8f0;
+        border-radius: 12px;
+        padding: 16px;
+      }
+      .tile.bad {
+        border-color: #e53e3e;
+        background: #fff5f5;
+      }
+      .tile-value {
+        font-size: 1.8rem;
+        font-weight: 600;
+      }
+      .tile-label {
+        color: #2d3748;
+        margin-top: 4px;
+      }
+      .tile-hint {
+        font-size: 0.8rem;
+        color: #718096;
+        margin-top: 4px;
+      }
+      table.docs {
+        width: 100%;
+        border-collapse: collapse;
+        background: #fff;
+        border: 1px solid #e2e8f0;
+        border-radius: 12px;
+        overflow: hidden;
+      }
+      .docs th,
+      .docs td {
+        text-align: left;
+        padding: 10px 12px;
+        border-bottom: 1px solid #edf2f7;
+        font-size: 0.9rem;
+        vertical-align: top;
+      }
+      .docs th {
+        background: #f7fafc;
+        color: #4a5568;
+        font-weight: 600;
+      }
+      .docs td.fail {
+        color: #c53030;
+      }
+      .docs tr:last-child td {
+        border-bottom: 0;
+      }
+    `,
+  ],
+})
+export class EvalsComponent implements OnInit {
+  report: EvalReport | null = null;
+  loaded = false;
+  error = '';
+
+  constructor(private agent: AgentService) {}
+
+  ngOnInit(): void {
+    this.agent.evals().subscribe({
+      next: (report) => {
+        this.report = report;
+        this.loaded = true;
+      },
+      error: (e) => {
+        this.error = `Couldn't load evals: ${e?.message ?? e}`;
+        this.loaded = true;
+      },
+    });
+  }
+}

docker-compose.yml

@@ -82,6 +82,9 @@ services:
         condition: service_completed_successfully
     volumes:
       - ./examples:/app/examples:ro
+      # Writable so `docker compose exec api python evals/run_evals.py --out
+      # evals/results.json` can produce a report the Evals view then serves.
+      - ./evals:/app/evals
 
   console:
     build: ./apps/console

docs/SMOKE_TEST.md

@@ -144,6 +144,9 @@ Open <http://localhost:4200>.
       an error rather than hanging.
 - [ ] Click **Knowledge** — a table lists the ingested sources (AML/KYC policy,
       product sheet) with their chunk counts.
+- [ ] Click **Evals**. Before a run it shows a "not run yet" note; after
+      `docker compose exec api python evals/run_evals.py --out evals/results.json`
+      it shows the pass rate, gate PASS/FAIL, and a per-case table.
 - [ ] Click **Operations** — tiles show live counts (chat requests, grounded %,
       approvals, PII redactions, HTTP requests) that update as you use Chat.