diff --git a/.env.example b/.env.example index 8cc448e..a610c5c 100644 --- a/.env.example +++ b/.env.example @@ -39,9 +39,14 @@ REDACT_PII=true OBSERVABILITY_BACKEND=none LANGCHAIN_API_KEY= LANGCHAIN_PROJECT=agentforge +# Langfuse: keys from your project (cloud or self-hosted). With the self-host +# overlay (docker-compose.langfuse.yml), HOST is set to http://langfuse:3000. LANGFUSE_PUBLIC_KEY= LANGFUSE_SECRET_KEY= LANGFUSE_HOST=https://cloud.langfuse.com +# Only for the self-hosted Langfuse server (docker-compose.langfuse.yml). +LANGFUSE_NEXTAUTH_SECRET= +LANGFUSE_SALT= # API gateway. API_HOST=0.0.0.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f53ee1d..e8a08f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,8 @@ jobs: - uses: actions/checkout@v5 - name: docker compose config run: docker compose config --quiet + - name: docker compose config (+ langfuse overlay) + run: docker compose -f docker-compose.yml -f docker-compose.langfuse.yml config --quiet # Validate the Kubernetes manifests render and pass schema validation. k8s-validate: diff --git a/Dockerfile b/Dockerfile index d7bd114..31981f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,12 @@ LABEL org.opencontainers.image.description="AgentForge API gateway" WORKDIR /app -# Install dependencies first (cached layer). +# Install dependencies first (cached layer). Optional extras can be baked in at +# build time, e.g. --build-arg EXTRAS=langfuse (used by docker-compose.langfuse.yml). +ARG EXTRAS="" COPY pyproject.toml README.md ./ COPY agentforge ./agentforge -RUN pip install --upgrade pip && pip install . +RUN pip install --upgrade pip && pip install ".${EXTRAS:+[${EXTRAS}]}" # Bundle the reference corpus so the image is self-contained for ingest # (compose bind-mounts ./examples over this; k8s/standalone runs rely on it). diff --git a/README.md b/README.md index 1fd6cd2..23bf3d4 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ A **banking compliance assistant** ships as the reference example — RAG over p - **RAG pipeline** — ingestion, chunking, embeddings, retrieval with citations and grounded refusals. - **Agentic orchestration** — stateful LangGraph agents with durable execution and human-in-the-loop approval nodes. - **Guardrails** — PII redaction, tool scoping, and policy enforcement via LangChain middleware. -- **Pluggable observability** — LangSmith by default; Langfuse adapter for a fully self-hosted setup; Prometheus `/metrics`. +- **Pluggable observability** — LangSmith by default; self-hostable Langfuse adapter (bundled compose overlay); Prometheus `/metrics`. See [`docs/OBSERVABILITY.md`](docs/OBSERVABILITY.md). - **Evals as a CI gate** — regression tests on retrieval quality and answer faithfulness that block bad deploys. - **Admin console** — Angular UI: chat, the approval queue, knowledge base, eval scores, and live ops metrics. - **One-command local run** — `docker compose up` to a working agent + console. diff --git a/agentforge/observability/tracing.py b/agentforge/observability/tracing.py index f813270..0f9b691 100644 --- a/agentforge/observability/tracing.py +++ b/agentforge/observability/tracing.py @@ -28,24 +28,47 @@ def setup_observability() -> None: os.environ["LANGCHAIN_API_KEY"] = settings.langchain_api_key os.environ["LANGCHAIN_PROJECT"] = settings.langchain_project + elif backend == "langfuse": + # Export the standard LANGFUSE_* vars so the SDK/handler pick them up + # (works against Langfuse cloud or a self-hosted instance via the host). + for var, value in ( + ("LANGFUSE_PUBLIC_KEY", settings.langfuse_public_key), + ("LANGFUSE_SECRET_KEY", settings.langfuse_secret_key), + ("LANGFUSE_HOST", settings.langfuse_host), + ): + if value: + os.environ[var] = value + @lru_cache def get_callbacks() -> list[Any]: - """Run callbacks for the active backend (empty for langsmith/none).""" + """Run callbacks for the active backend (empty for langsmith/none). + + For langfuse, return its LangChain ``CallbackHandler`` so every node/LLM call + is traced. Auth comes from the ``LANGFUSE_*`` env exported by + ``setup_observability`` (or already in the environment). + """ settings = get_settings() - if settings.observability_backend.lower() == "langfuse": - try: - from langfuse.callback import CallbackHandler - except ImportError as exc: # pragma: no cover - optional dependency - raise RuntimeError( - "OBSERVABILITY_BACKEND=langfuse requires the 'langfuse' extra: " - "pip install 'agentforge[langfuse]'" - ) from exc - return [ - CallbackHandler( - public_key=settings.langfuse_public_key, - secret_key=settings.langfuse_secret_key, - host=settings.langfuse_host, - ) - ] - return [] + if settings.observability_backend.lower() != "langfuse": + return [] + + try: + from langfuse.callback import CallbackHandler + except ImportError as exc: # pragma: no cover - optional dependency + raise RuntimeError( + "OBSERVABILITY_BACKEND=langfuse requires the 'langfuse' extra: " + "pip install 'agentforge[langfuse]'" + ) from exc + + # Pass explicit credentials when configured; otherwise the handler falls back + # to the LANGFUSE_* environment. + kwargs = { + k: v + for k, v in ( + ("public_key", settings.langfuse_public_key), + ("secret_key", settings.langfuse_secret_key), + ("host", settings.langfuse_host), + ) + if v + } + return [CallbackHandler(**kwargs)] diff --git a/docker-compose.langfuse.yml b/docker-compose.langfuse.yml new file mode 100644 index 0000000..7d14ad8 --- /dev/null +++ b/docker-compose.langfuse.yml @@ -0,0 +1,59 @@ +# Fully self-hosted observability: adds a Langfuse server (+ its own Postgres) +# and points the API's tracing at it. Use it as an overlay on the base compose: +# +# docker compose -f docker-compose.yml -f docker-compose.langfuse.yml up --build +# +# Then open http://localhost:3000, create an account + project, copy the project +# keys into your .env as LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY, and restart +# the api. Traces for every graph node appear in the Langfuse project. +# +# `--build` is required the first time so the api image is rebuilt with the +# langfuse extra (EXTRAS=langfuse). + +services: + langfuse-db: + image: postgres:16 + environment: + POSTGRES_USER: langfuse + POSTGRES_PASSWORD: langfuse + POSTGRES_DB: langfuse + volumes: + - langfuse_pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U langfuse"] + interval: 5s + timeout: 5s + retries: 10 + + langfuse: + image: langfuse/langfuse:2 + depends_on: + langfuse-db: + condition: service_healthy + environment: + DATABASE_URL: postgresql://langfuse:langfuse@langfuse-db:5432/langfuse + NEXTAUTH_URL: http://localhost:3000 + # Dev defaults — override in .env for anything beyond a local trial. + NEXTAUTH_SECRET: ${LANGFUSE_NEXTAUTH_SECRET:-dev-nextauth-secret-change-me} + SALT: ${LANGFUSE_SALT:-dev-salt-change-me} + TELEMETRY_ENABLED: "false" + ports: + - "3000:3000" + + api: + build: + context: . + args: + EXTRAS: langfuse + environment: + OBSERVABILITY_BACKEND: langfuse + LANGFUSE_HOST: http://langfuse:3000 + # Supplied via .env after you create the project in the Langfuse UI. + LANGFUSE_PUBLIC_KEY: ${LANGFUSE_PUBLIC_KEY:-} + LANGFUSE_SECRET_KEY: ${LANGFUSE_SECRET_KEY:-} + depends_on: + langfuse: + condition: service_started + +volumes: + langfuse_pgdata: diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md new file mode 100644 index 0000000..19522cb --- /dev/null +++ b/docs/OBSERVABILITY.md @@ -0,0 +1,79 @@ +# Observability + +Every graph node and LLM call can be traced. One switch — `OBSERVABILITY_BACKEND` +— selects where traces go. Callbacks are attached to each run in the API +(`_run_config`), so tracing is uniform across `/chat`, `/chat/stream`, and +`/approve`. + +| `OBSERVABILITY_BACKEND` | Where traces go | Extra install | +|---|---|---| +| `none` (default) | nowhere (local dev / tests) | — | +| `langsmith` | LangSmith (hosted) | — | +| `langfuse` | Langfuse (cloud **or** self-hosted) | `agentforge[langfuse]` | + +For runtime metrics (request rates, grounded-answer ratio, approvals, …) see the +Prometheus `/metrics` endpoint and the console's Operations tab instead — that's +complementary to the per-trace view here. + +## LangSmith + +```env +OBSERVABILITY_BACKEND=langsmith +LANGCHAIN_API_KEY=ls-... +LANGCHAIN_PROJECT=agentforge +``` + +`setup_observability()` exports the `LANGCHAIN_*` env at startup and LangChain +auto-traces every node — no per-call wiring. + +## Langfuse — cloud + +Create a project at https://cloud.langfuse.com, then: + +```env +OBSERVABILITY_BACKEND=langfuse +LANGFUSE_PUBLIC_KEY=pk-lf-... +LANGFUSE_SECRET_KEY=sk-lf-... +LANGFUSE_HOST=https://cloud.langfuse.com +``` + +The API image must include the extra. Locally that's `pip install +'agentforge[langfuse]'`; in containers, build with `--build-arg EXTRAS=langfuse` +(the self-host overlay below does this for you). + +## Langfuse — fully self-hosted + +[`docker-compose.langfuse.yml`](../docker-compose.langfuse.yml) adds a Langfuse +server and its own Postgres, and points the API at it. Bring up the stack with +the overlay: + +```bash +docker compose -f docker-compose.yml -f docker-compose.langfuse.yml up --build +``` + +`--build` rebuilds the api image with the `langfuse` extra the first time. Then: + +1. Open , create an account + a project. +2. Copy the project's keys into `.env`: + ```env + LANGFUSE_PUBLIC_KEY=pk-lf-... + LANGFUSE_SECRET_KEY=sk-lf-... + ``` +3. Restart the api: `docker compose ... up -d api`. +4. Ask a question, then refresh the Langfuse project — a trace per graph node + (guardrails → retrieve → generate → …) appears. + +`LANGFUSE_HOST` is already set to `http://langfuse:3000` by the overlay. Set +`LANGFUSE_NEXTAUTH_SECRET` and `LANGFUSE_SALT` in `.env` for anything beyond a +local trial. + +> Uses Langfuse v2 (single container + Postgres) to keep self-hosting light; the +> `langfuse` extra is pinned to match. Langfuse v3 self-hosting (ClickHouse + +> Redis + object storage) is out of scope for this overlay — point `LANGFUSE_HOST` +> at such an instance and bump the extra if you run one. + +## Kubernetes + +The manifests don't bundle Langfuse. Run it (or LangSmith) out of band and set +`OBSERVABILITY_BACKEND` + the `LANGFUSE_*` / `LANGCHAIN_*` keys in the Secret; +for Langfuse, build/push an api image with `EXTRAS=langfuse`. diff --git a/docs/SMOKE_TEST.md b/docs/SMOKE_TEST.md index 9b1ee86..c4f061e 100644 --- a/docs/SMOKE_TEST.md +++ b/docs/SMOKE_TEST.md @@ -176,6 +176,16 @@ LANGCHAIN_API_KEY=ls-... - [ ] After a chat, traces for each graph node appear in the LangSmith project. +For a fully self-hosted alternative, bring up the bundled Langfuse server and +trace into it — see [`docs/OBSERVABILITY.md`](OBSERVABILITY.md): + +```bash +docker compose -f docker-compose.yml -f docker-compose.langfuse.yml up --build +``` + +- [ ] After wiring the project keys, traces for each graph node appear at + . + --- ## Teardown diff --git a/pyproject.toml b/pyproject.toml index 5b87ca3..c54a2cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,9 @@ dependencies = [ ] [project.optional-dependencies] -# Fully self-hosted observability backend. -langfuse = ["langfuse>=2.0"] +# Fully self-hosted observability backend. Pinned to v2 to match the LangChain +# CallbackHandler API and the single-container self-host in docker-compose.langfuse.yml. +langfuse = ["langfuse>=2.0,<3.0"] # Local embeddings (sentence-transformers) — avoids any external embedding API. local-embeddings = ["langchain-huggingface>=0.1", "sentence-transformers>=3.0"] dev = [ diff --git a/tests/test_observability.py b/tests/test_observability.py new file mode 100644 index 0000000..dfcce49 --- /dev/null +++ b/tests/test_observability.py @@ -0,0 +1,65 @@ +"""Observability backend selection. No external services or langfuse install.""" + +from __future__ import annotations + +import os + +import pytest + +from agentforge.config import get_settings +from agentforge.observability import tracing + + +@pytest.fixture(autouse=True) +def _clear_callbacks_cache(): + # get_callbacks is lru_cached; reset around each test so backend changes apply. + tracing.get_callbacks.cache_clear() + yield + tracing.get_callbacks.cache_clear() + + +def test_no_callbacks_for_none_or_langsmith(monkeypatch): + settings = get_settings() + monkeypatch.setattr(settings, "observability_backend", "none") + assert tracing.get_callbacks() == [] + + tracing.get_callbacks.cache_clear() + monkeypatch.setattr(settings, "observability_backend", "langsmith") + assert tracing.get_callbacks() == [] + + +def test_langfuse_backend_requires_extra(monkeypatch): + # langfuse isn't in the dev install, so selecting it must raise a helpful error. + monkeypatch.setattr(get_settings(), "observability_backend", "langfuse") + with pytest.raises(RuntimeError, match="langfuse"): + tracing.get_callbacks() + + +def test_setup_langsmith_exports_env(monkeypatch): + settings = get_settings() + monkeypatch.setattr(settings, "observability_backend", "langsmith") + monkeypatch.setattr(settings, "langchain_api_key", "ls-test") + monkeypatch.delenv("LANGCHAIN_TRACING_V2", raising=False) + + tracing.setup_observability() + assert os.environ["LANGCHAIN_TRACING_V2"] == "true" + assert os.environ["LANGCHAIN_API_KEY"] == "ls-test" + + for key in ("LANGCHAIN_TRACING_V2", "LANGCHAIN_API_KEY"): + os.environ.pop(key, None) + + +def test_setup_langfuse_exports_env(monkeypatch): + settings = get_settings() + monkeypatch.setattr(settings, "observability_backend", "langfuse") + monkeypatch.setattr(settings, "langfuse_public_key", "pk-test") + monkeypatch.setattr(settings, "langfuse_secret_key", "sk-test") + monkeypatch.setattr(settings, "langfuse_host", "http://langfuse:3000") + + tracing.setup_observability() + assert os.environ["LANGFUSE_PUBLIC_KEY"] == "pk-test" + assert os.environ["LANGFUSE_SECRET_KEY"] == "sk-test" + assert os.environ["LANGFUSE_HOST"] == "http://langfuse:3000" + + for key in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", "LANGFUSE_HOST"): + os.environ.pop(key, None)