Langfuse adapter: self-hostable tracing backend

.env.example

@@ -39,9 +39,14 @@ REDACT_PII=true
 OBSERVABILITY_BACKEND=none
 LANGCHAIN_API_KEY=
 LANGCHAIN_PROJECT=agentforge
+# Langfuse: keys from your project (cloud or self-hosted). With the self-host
+# overlay (docker-compose.langfuse.yml), HOST is set to http://langfuse:3000.
 LANGFUSE_PUBLIC_KEY=
 LANGFUSE_SECRET_KEY=
 LANGFUSE_HOST=https://cloud.langfuse.com
+# Only for the self-hosted Langfuse server (docker-compose.langfuse.yml).
+LANGFUSE_NEXTAUTH_SECRET=
+LANGFUSE_SALT=
 
 # API gateway.
 API_HOST=0.0.0.0

.github/workflows/ci.yml

@@ -28,6 +28,8 @@ jobs:
       - uses: actions/checkout@v5
       - name: docker compose config
         run: docker compose config --quiet
+      - name: docker compose config (+ langfuse overlay)
+        run: docker compose -f docker-compose.yml -f docker-compose.langfuse.yml config --quiet
 
   # Validate the Kubernetes manifests render and pass schema validation.
   k8s-validate:

Dockerfile

@@ -10,10 +10,12 @@ LABEL org.opencontainers.image.description="AgentForge API gateway"
 
 WORKDIR /app
 
-# Install dependencies first (cached layer).
+# Install dependencies first (cached layer). Optional extras can be baked in at
+# build time, e.g. --build-arg EXTRAS=langfuse (used by docker-compose.langfuse.yml).
+ARG EXTRAS=""
 COPY pyproject.toml README.md ./
 COPY agentforge ./agentforge
-RUN pip install --upgrade pip && pip install .
+RUN pip install --upgrade pip && pip install ".${EXTRAS:+[${EXTRAS}]}"
 
 # Bundle the reference corpus so the image is self-contained for ingest
 # (compose bind-mounts ./examples over this; k8s/standalone runs rely on it).

README.md

@@ -23,7 +23,7 @@ A **banking compliance assistant** ships as the reference example — RAG over p
 - **RAG pipeline** — ingestion, chunking, embeddings, retrieval with citations and grounded refusals.
 - **Agentic orchestration** — stateful LangGraph agents with durable execution and human-in-the-loop approval nodes.
 - **Guardrails** — PII redaction, tool scoping, and policy enforcement via LangChain middleware.
-- **Pluggable observability** — LangSmith by default; Langfuse adapter for a fully self-hosted setup; Prometheus `/metrics`.
+- **Pluggable observability** — LangSmith by default; self-hostable Langfuse adapter (bundled compose overlay); Prometheus `/metrics`. See [`docs/OBSERVABILITY.md`](docs/OBSERVABILITY.md).
 - **Evals as a CI gate** — regression tests on retrieval quality and answer faithfulness that block bad deploys.
 - **Admin console** — Angular UI: chat, the approval queue, knowledge base, eval scores, and live ops metrics.
 - **One-command local run** — `docker compose up` to a working agent + console.

agentforge/observability/tracing.py

@@ -28,24 +28,47 @@ def setup_observability() -> None:
         os.environ["LANGCHAIN_API_KEY"] = settings.langchain_api_key
         os.environ["LANGCHAIN_PROJECT"] = settings.langchain_project
 
+    elif backend == "langfuse":
+        # Export the standard LANGFUSE_* vars so the SDK/handler pick them up
+        # (works against Langfuse cloud or a self-hosted instance via the host).
+        for var, value in (
+            ("LANGFUSE_PUBLIC_KEY", settings.langfuse_public_key),
+            ("LANGFUSE_SECRET_KEY", settings.langfuse_secret_key),
+            ("LANGFUSE_HOST", settings.langfuse_host),
+        ):
+            if value:
+                os.environ[var] = value
+
 
 @lru_cache
 def get_callbacks() -> list[Any]:
-    """Run callbacks for the active backend (empty for langsmith/none)."""
+    """Run callbacks for the active backend (empty for langsmith/none).
+
+    For langfuse, return its LangChain ``CallbackHandler`` so every node/LLM call
+    is traced. Auth comes from the ``LANGFUSE_*`` env exported by
+    ``setup_observability`` (or already in the environment).
+    """
     settings = get_settings()
-    if settings.observability_backend.lower() == "langfuse":
-        try:
-            from langfuse.callback import CallbackHandler
-        except ImportError as exc:  # pragma: no cover - optional dependency
-            raise RuntimeError(
-                "OBSERVABILITY_BACKEND=langfuse requires the 'langfuse' extra: "
-                "pip install 'agentforge[langfuse]'"
-            ) from exc
-        return [
-            CallbackHandler(
-                public_key=settings.langfuse_public_key,
-                secret_key=settings.langfuse_secret_key,
-                host=settings.langfuse_host,
-            )
-        ]
-    return []
+    if settings.observability_backend.lower() != "langfuse":
+        return []
+
+    try:
+        from langfuse.callback import CallbackHandler
+    except ImportError as exc:  # pragma: no cover - optional dependency
+        raise RuntimeError(
+            "OBSERVABILITY_BACKEND=langfuse requires the 'langfuse' extra: "
+            "pip install 'agentforge[langfuse]'"
+        ) from exc
+
+    # Pass explicit credentials when configured; otherwise the handler falls back
+    # to the LANGFUSE_* environment.
+    kwargs = {
+        k: v
+        for k, v in (
+            ("public_key", settings.langfuse_public_key),
+            ("secret_key", settings.langfuse_secret_key),
+            ("host", settings.langfuse_host),
+        )
+        if v
+    }
+    return [CallbackHandler(**kwargs)]

docker-compose.langfuse.yml

@@ -0,0 +1,59 @@
+# Fully self-hosted observability: adds a Langfuse server (+ its own Postgres)
+# and points the API's tracing at it. Use it as an overlay on the base compose:
+#
+#   docker compose -f docker-compose.yml -f docker-compose.langfuse.yml up --build
+#
+# Then open http://localhost:3000, create an account + project, copy the project
+# keys into your .env as LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY, and restart
+# the api. Traces for every graph node appear in the Langfuse project.
+#
+# `--build` is required the first time so the api image is rebuilt with the
+# langfuse extra (EXTRAS=langfuse).
+
+services:
+  langfuse-db:
+    image: postgres:16
+    environment:
+      POSTGRES_USER: langfuse
+      POSTGRES_PASSWORD: langfuse
+      POSTGRES_DB: langfuse
+    volumes:
+      - langfuse_pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U langfuse"]
+      interval: 5s
+      timeout: 5s
+      retries: 10
+
+  langfuse:
+    image: langfuse/langfuse:2
+    depends_on:
+      langfuse-db:
+        condition: service_healthy
+    environment:
+      DATABASE_URL: postgresql://langfuse:langfuse@langfuse-db:5432/langfuse
+      NEXTAUTH_URL: http://localhost:3000
+      # Dev defaults — override in .env for anything beyond a local trial.
+      NEXTAUTH_SECRET: ${LANGFUSE_NEXTAUTH_SECRET:-dev-nextauth-secret-change-me}
+      SALT: ${LANGFUSE_SALT:-dev-salt-change-me}
+      TELEMETRY_ENABLED: "false"
+    ports:
+      - "3000:3000"
+
+  api:
+    build:
+      context: .
+      args:
+        EXTRAS: langfuse
+    environment:
+      OBSERVABILITY_BACKEND: langfuse
+      LANGFUSE_HOST: http://langfuse:3000
+      # Supplied via .env after you create the project in the Langfuse UI.
+      LANGFUSE_PUBLIC_KEY: ${LANGFUSE_PUBLIC_KEY:-}
+      LANGFUSE_SECRET_KEY: ${LANGFUSE_SECRET_KEY:-}
+    depends_on:
+      langfuse:
+        condition: service_started
+
+volumes:
+  langfuse_pgdata:

docs/OBSERVABILITY.md

@@ -0,0 +1,79 @@
+# Observability
+
+Every graph node and LLM call can be traced. One switch — `OBSERVABILITY_BACKEND`
+— selects where traces go. Callbacks are attached to each run in the API
+(`_run_config`), so tracing is uniform across `/chat`, `/chat/stream`, and
+`/approve`.
+
+| `OBSERVABILITY_BACKEND` | Where traces go | Extra install |
+|---|---|---|
+| `none` (default) | nowhere (local dev / tests) | — |
+| `langsmith` | LangSmith (hosted) | — |
+| `langfuse` | Langfuse (cloud **or** self-hosted) | `agentforge[langfuse]` |
+
+For runtime metrics (request rates, grounded-answer ratio, approvals, …) see the
+Prometheus `/metrics` endpoint and the console's Operations tab instead — that's
+complementary to the per-trace view here.
+
+## LangSmith
+
+```env
+OBSERVABILITY_BACKEND=langsmith
+LANGCHAIN_API_KEY=ls-...
+LANGCHAIN_PROJECT=agentforge
+```
+
+`setup_observability()` exports the `LANGCHAIN_*` env at startup and LangChain
+auto-traces every node — no per-call wiring.
+
+## Langfuse — cloud
+
+Create a project at https://cloud.langfuse.com, then:
+
+```env
+OBSERVABILITY_BACKEND=langfuse
+LANGFUSE_PUBLIC_KEY=pk-lf-...
+LANGFUSE_SECRET_KEY=sk-lf-...
+LANGFUSE_HOST=https://cloud.langfuse.com
+```
+
+The API image must include the extra. Locally that's `pip install
+'agentforge[langfuse]'`; in containers, build with `--build-arg EXTRAS=langfuse`
+(the self-host overlay below does this for you).
+
+## Langfuse — fully self-hosted
+
+[`docker-compose.langfuse.yml`](../docker-compose.langfuse.yml) adds a Langfuse
+server and its own Postgres, and points the API at it. Bring up the stack with
+the overlay:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.langfuse.yml up --build
+```
+
+`--build` rebuilds the api image with the `langfuse` extra the first time. Then:
+
+1. Open <http://localhost:3000>, create an account + a project.
+2. Copy the project's keys into `.env`:
+   ```env
+   LANGFUSE_PUBLIC_KEY=pk-lf-...
+   LANGFUSE_SECRET_KEY=sk-lf-...
+   ```
+3. Restart the api: `docker compose ... up -d api`.
+4. Ask a question, then refresh the Langfuse project — a trace per graph node
+   (guardrails → retrieve → generate → …) appears.
+
+`LANGFUSE_HOST` is already set to `http://langfuse:3000` by the overlay. Set
+`LANGFUSE_NEXTAUTH_SECRET` and `LANGFUSE_SALT` in `.env` for anything beyond a
+local trial.
+
+> Uses Langfuse v2 (single container + Postgres) to keep self-hosting light; the
+> `langfuse` extra is pinned to match. Langfuse v3 self-hosting (ClickHouse +
+> Redis + object storage) is out of scope for this overlay — point `LANGFUSE_HOST`
+> at such an instance and bump the extra if you run one.
+
+## Kubernetes
+
+The manifests don't bundle Langfuse. Run it (or LangSmith) out of band and set
+`OBSERVABILITY_BACKEND` + the `LANGFUSE_*` / `LANGCHAIN_*` keys in the Secret;
+for Langfuse, build/push an api image with `EXTRAS=langfuse`.

docs/SMOKE_TEST.md

@@ -176,6 +176,16 @@ LANGCHAIN_API_KEY=ls-...
 
 - [ ] After a chat, traces for each graph node appear in the LangSmith project.
 
+For a fully self-hosted alternative, bring up the bundled Langfuse server and
+trace into it — see [`docs/OBSERVABILITY.md`](OBSERVABILITY.md):
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.langfuse.yml up --build
+```
+
+- [ ] After wiring the project keys, traces for each graph node appear at
+      <http://localhost:3000>.
+
 ---
 
 ## Teardown

pyproject.toml

@@ -41,8 +41,9 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-# Fully self-hosted observability backend.
-langfuse = ["langfuse>=2.0"]
+# Fully self-hosted observability backend. Pinned to v2 to match the LangChain
+# CallbackHandler API and the single-container self-host in docker-compose.langfuse.yml.
+langfuse = ["langfuse>=2.0,<3.0"]
 # Local embeddings (sentence-transformers) — avoids any external embedding API.
 local-embeddings = ["langchain-huggingface>=0.1", "sentence-transformers>=3.0"]
 dev = [

tests/test_observability.py

@@ -0,0 +1,65 @@
+"""Observability backend selection. No external services or langfuse install."""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from agentforge.config import get_settings
+from agentforge.observability import tracing
+
+
+@pytest.fixture(autouse=True)
+def _clear_callbacks_cache():
+    # get_callbacks is lru_cached; reset around each test so backend changes apply.
+    tracing.get_callbacks.cache_clear()
+    yield
+    tracing.get_callbacks.cache_clear()
+
+
+def test_no_callbacks_for_none_or_langsmith(monkeypatch):
+    settings = get_settings()
+    monkeypatch.setattr(settings, "observability_backend", "none")
+    assert tracing.get_callbacks() == []
+
+    tracing.get_callbacks.cache_clear()
+    monkeypatch.setattr(settings, "observability_backend", "langsmith")
+    assert tracing.get_callbacks() == []
+
+
+def test_langfuse_backend_requires_extra(monkeypatch):
+    # langfuse isn't in the dev install, so selecting it must raise a helpful error.
+    monkeypatch.setattr(get_settings(), "observability_backend", "langfuse")
+    with pytest.raises(RuntimeError, match="langfuse"):
+        tracing.get_callbacks()
+
+
+def test_setup_langsmith_exports_env(monkeypatch):
+    settings = get_settings()
+    monkeypatch.setattr(settings, "observability_backend", "langsmith")
+    monkeypatch.setattr(settings, "langchain_api_key", "ls-test")
+    monkeypatch.delenv("LANGCHAIN_TRACING_V2", raising=False)
+
+    tracing.setup_observability()
+    assert os.environ["LANGCHAIN_TRACING_V2"] == "true"
+    assert os.environ["LANGCHAIN_API_KEY"] == "ls-test"
+
+    for key in ("LANGCHAIN_TRACING_V2", "LANGCHAIN_API_KEY"):
+        os.environ.pop(key, None)
+
+
+def test_setup_langfuse_exports_env(monkeypatch):
+    settings = get_settings()
+    monkeypatch.setattr(settings, "observability_backend", "langfuse")
+    monkeypatch.setattr(settings, "langfuse_public_key", "pk-test")
+    monkeypatch.setattr(settings, "langfuse_secret_key", "sk-test")
+    monkeypatch.setattr(settings, "langfuse_host", "http://langfuse:3000")
+
+    tracing.setup_observability()
+    assert os.environ["LANGFUSE_PUBLIC_KEY"] == "pk-test"
+    assert os.environ["LANGFUSE_SECRET_KEY"] == "sk-test"
+    assert os.environ["LANGFUSE_HOST"] == "http://langfuse:3000"
+
+    for key in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_HOST"):
+        os.environ.pop(key, None)