feat: native OpenTelemetry instrumentation (optional, off by default)

.claude/rules/logging-observability.md

@@ -21,3 +21,11 @@ paths:
 - **Metrics** go through `core.observability.metrics` (Prometheus); don't invent
   ad-hoc counters. Histograms/counters/gauges have registry helpers.
 - Don't log secrets, API keys, or full memory content at `info`/above.
+- **Tracing** (optional, `[otel]` extra, **off by default**): open spans with
+  `memory_span(...)` from `core.observability.tracing` — it stamps the Langfuse
+  `langfuse.*` attributes and is a no-op until `[observability] enabled`, so call
+  sites never branch on config. LLM / embedding token usage rides
+  `set_generation_usage` onto the active span (Langfuse computes cost).
+  Request/response content is emitted only when `capture_content` is on
+  (redaction hook + truncation). `request_id` is kept independent of the OTel
+  `trace_id`; an upstream `traceparent` header is continued when present.

pyproject.toml

@@ -70,6 +70,12 @@ dependencies = [
 
 [project.optional-dependencies]
 multimodal = ["everalgo-parser[svg]>=0.2.1"]  # [svg] bundles cairosvg → SVG works by default
+# Native OpenTelemetry tracing export. Optional — EverOS never imports these
+# unless [observability] is enabled. Install with: pip install everos[otel]
+otel = [
+    "opentelemetry-sdk>=1.27.0",
+    "opentelemetry-exporter-otlp-proto-http>=1.27.0",
+]
 
 [project.urls]
 Homepage = "https://evermind.ai"
@@ -255,4 +261,8 @@ dev = [
     "pre-commit>=4.0.0",
     "ipdb>=0.13.13",
     "pyinstrument>=5.0.0",
+    # Tracing tests must actually run (no skip-when-absent), so the optional
+    # [otel] stack is always present in the dev / CI environment.
+    "opentelemetry-sdk>=1.27.0",
+    "opentelemetry-exporter-otlp-proto-http>=1.27.0",
 ]

src/everos/component/embedding/openai_provider.py

@@ -23,6 +23,8 @@
 
 import openai
 
+from everos.core.observability.tracing import set_generation_usage
+
 from .protocol import EmbeddingServiceError
 
 
@@ -94,5 +96,12 @@ async def _embed_chunk(self, chunk: list[str]) -> list[list[float]]:
                 )
             except openai.OpenAIError as exc:
                 raise EmbeddingServiceError(str(exc)) from exc
+        # Surface token usage onto the active span (e.g. everos.search.embed_query).
+        # No-op when tracing is off; embeddings report only input (prompt) tokens.
+        usage = getattr(response, "usage", None)
+        set_generation_usage(
+            model=self._model,
+            input_tokens=usage.prompt_tokens if usage else None,
+        )
         # OpenAI returns ``data`` indexed by request order; truncate to ``dim``.
         return [list(item.embedding[: self.dim]) for item in response.data]

src/everos/component/llm/_usage_client.py

@@ -0,0 +1,62 @@
+"""Token-usage-recording LLM client wrapper.
+
+Wraps any :class:`everalgo.llm.LLMClient` and, after each ``chat`` call,
+writes ``response.usage`` (+ model) onto the current OpenTelemetry span via
+``set_generation_usage``. The wrapped client's behaviour is otherwise
+untouched — the response is returned verbatim and all other attributes
+delegate through.
+
+This is how token counts reach the ``everos.extract`` generation span:
+everalgo's extractors own the ``chat`` call and discard the ``ChatResponse``
+(keeping only ``.content``), so usage is captured here at the client
+boundary — no everalgo change required. When no span is active (e.g. the
+search path, or tracing disabled), ``set_generation_usage`` no-ops.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from everos.core.observability.tracing import set_generation_usage
+
+if TYPE_CHECKING:
+    from everalgo.llm import ChatMessage, ChatResponse
+    from everalgo.llm.protocols import LLMClient
+    from pydantic import BaseModel
+
+
+class UsageRecordingClient:
+    """LLM client proxy that records token usage onto the active span."""
+
+    def __init__(self, inner: LLMClient) -> None:
+        self._inner = inner
+
+    async def chat(
+        self,
+        messages: list[ChatMessage],
+        *,
+        model: str | None = None,
+        temperature: float | None = None,
+        max_tokens: int | None = None,
+        response_format: type[BaseModel] | None = None,
+        **extra: Any,
+    ) -> ChatResponse:
+        response = await self._inner.chat(
+            messages,
+            model=model,
+            temperature=temperature,
+            max_tokens=max_tokens,
+            response_format=response_format,
+            **extra,
+        )
+        usage = response.usage
+        set_generation_usage(
+            model=response.model,
+            input_tokens=usage.prompt_tokens if usage else None,
+            output_tokens=usage.completion_tokens if usage else None,
+        )
+        return response
+
+    def __getattr__(self, name: str) -> Any:
+        # Delegate everything else to the wrapped client.
+        return getattr(self._inner, name)

src/everos/component/llm/client.py

@@ -16,6 +16,8 @@
 from everos.config import load_settings
 from everos.core.observability.logging import get_logger
 
+from ._usage_client import UsageRecordingClient
+
 logger = get_logger(__name__)
 
 
@@ -38,21 +40,27 @@ def get_llm_client() -> LLMClient:
     if _llm_client is not None:
         return _llm_client
 
-    llm_cfg = load_settings().llm
+    settings = load_settings()
+    llm_cfg = settings.llm
     api_key = (
         llm_cfg.api_key.get_secret_value() if llm_cfg.api_key is not None else None
     )
     if not api_key or not llm_cfg.base_url:
         raise LLMNotConfiguredError(
             "LLM is required; set EVEROS_LLM__API_KEY + EVEROS_LLM__BASE_URL"
         )
-    _llm_client = build_client(
+    client: LLMClient = build_client(
         LLMConfig(
             model=llm_cfg.model,
             api_key=api_key,
             base_url=llm_cfg.base_url,
         )
     )
+    # Wrap for OTel token capture only when tracing is on — keeps the
+    # disabled path (the default) allocation- and overhead-free.
+    if settings.observability.enabled:
+        client = UsageRecordingClient(client)
+    _llm_client = client
     logger.info("llm_client_built", model=llm_cfg.model)
     return _llm_client
 

src/everos/config/default.toml

@@ -145,3 +145,21 @@ session_lock_timeout_seconds = 360.0
 threshold = 0.65
 time_window_days = 7.0
 
+
+[observability]
+# OpenTelemetry tracing export. Off by default; pure OTLP/HTTP, vendor-neutral
+# (Langfuse, an OTel Collector, or any OTLP backend). EverOS ships no vendor SDK.
+# Override via EVEROS_OBSERVABILITY__ENABLED, EVEROS_OBSERVABILITY__ENDPOINT, etc.
+enabled = false
+exporter = "otlp_http"          # "otlp_http" | "none"
+endpoint = ""                    # e.g. https://us.cloud.langfuse.com/api/public/otel/v1/traces
+service_name = "everos"
+sample_rate = 1.0                # 0.0 to 1.0
+# Privacy: false (default) = metadata only; true also emits query / extracted
+# memory / .md paths as span input/output (redacted + truncated).
+capture_content = false
+# Recall-quality scores pushed to Langfuse (Langfuse-specific REST, off the
+# OTLP stream). Only fires when langfuse_public_key/secret_key/host are set
+# (via everos.toml or EVEROS_OBSERVABILITY__LANGFUSE_* — secrets, not shipped here).
+emit_recall_scores = true
+recall_hit_threshold = 0.6       # only meaningful for calibrated methods

src/everos/config/settings.py

@@ -350,6 +350,53 @@ class KnowledgeSettings(BaseModel):
     search: KnowledgeSearchSettings = KnowledgeSearchSettings()
 
 
+class ObservabilitySettings(BaseModel):
+    """``[observability]`` — OpenTelemetry tracing export.
+
+    Off by default. When ``enabled`` is true a ``TracerProvider`` is built
+    once at startup and standard OTLP/HTTP spans are exported to
+    ``endpoint``. The signal is pure OpenTelemetry — vendor-neutral — so it
+    works with any OTLP backend (Langfuse, an OTel Collector, ...); EverOS
+    does not depend on any vendor SDK.
+
+    ``langfuse_*`` are convenience credentials for pushing recall-quality
+    *scores* to Langfuse (a Langfuse-specific REST call, independent of the
+    OTLP span stream). Leave unset for a pure vendor-neutral OTLP export.
+
+    Env binding:
+        EVEROS_OBSERVABILITY__ENABLED
+        EVEROS_OBSERVABILITY__EXPORTER
+        EVEROS_OBSERVABILITY__ENDPOINT
+        EVEROS_OBSERVABILITY__SERVICE_NAME
+        EVEROS_OBSERVABILITY__SAMPLE_RATE
+        EVEROS_OBSERVABILITY__LANGFUSE_PUBLIC_KEY / __LANGFUSE_SECRET_KEY
+        EVEROS_OBSERVABILITY__LANGFUSE_HOST
+        EVEROS_OBSERVABILITY__EMIT_RECALL_SCORES
+        EVEROS_OBSERVABILITY__RECALL_HIT_THRESHOLD
+    """
+
+    enabled: bool = False
+    exporter: Literal["otlp_http", "none"] = "otlp_http"
+    endpoint: str = ""
+    headers: dict[str, str] = Field(default_factory=dict)
+    service_name: str = "everos"
+    sample_rate: float = Field(default=1.0, ge=0.0, le=1.0)
+    # Privacy: when False (default) spans carry metadata only — no query text,
+    # extracted memory, or .md paths. Set True to also emit request/response
+    # content as span input/output (redacted + truncated).
+    capture_content: bool = False
+
+    # Langfuse scores (recall-quality feedback) — optional, Langfuse-specific.
+    langfuse_public_key: str | None = None
+    langfuse_secret_key: SecretStr | None = None
+    langfuse_host: str | None = None
+    emit_recall_scores: bool = True
+    # ``hit`` threshold: only meaningful for calibrated-score methods
+    # (HYBRID LR / rerank / agentic). Not bounded to [0, 1] because raw
+    # BM25 scores are unbounded; tune per method on the eval side.
+    recall_hit_threshold: float = 0.6
+
+
 class Settings(BaseSettings):
     """Top-level application settings."""
 
@@ -365,6 +412,7 @@ class Settings(BaseSettings):
     clustering: ClusteringSettings = ClusteringSettings()
     multimodal: MultimodalSettings = MultimodalSettings()
     knowledge: KnowledgeSettings = KnowledgeSettings()
+    observability: ObservabilitySettings = ObservabilitySettings()
 
     model_config = SettingsConfigDict(
         env_prefix="EVEROS_",

src/everos/core/context/__init__.py

@@ -0,0 +1,22 @@
+"""core.context — request-scoped context propagation (contextvars).
+
+External usage::
+
+    from everos.core.context import (
+        get_request_id,
+        set_request_id,
+        reset_request_id,
+    )
+"""
+
+from .request import get_request_id as get_request_id
+from .request import reset_request_id as reset_request_id
+from .request import resolve_request_id as resolve_request_id
+from .request import set_request_id as set_request_id
+
+__all__ = [
+    "get_request_id",
+    "reset_request_id",
+    "resolve_request_id",
+    "set_request_id",
+]

src/everos/core/context/request.py

@@ -0,0 +1,39 @@
+"""Request-scoped context propagation via ``contextvars``.
+
+The request id is stored in a module-level ``ContextVar`` so it survives
+``await`` boundaries and is readable anywhere in the call chain (service,
+infra, log processors) without being threaded through call signatures.
+"""
+
+from __future__ import annotations
+
+from contextvars import ContextVar, Token
+
+from everos.core.observability.tracing import gen_request_id
+
+_request_id: ContextVar[str | None] = ContextVar("everos_request_id", default=None)
+
+
+def get_request_id() -> str | None:
+    """Return the request id bound to the current context, or ``None``."""
+    return _request_id.get()
+
+
+def set_request_id(value: str | None) -> Token[str | None]:
+    """Bind ``value`` as the current request id; return a reset token."""
+    return _request_id.set(value)
+
+
+def reset_request_id(token: Token[str | None]) -> None:
+    """Restore the request id to what it was before the matching ``set``."""
+    _request_id.reset(token)
+
+
+def resolve_request_id() -> str:
+    """Return the propagated request id, or mint a fresh W3C-compatible one.
+
+    Call sites that need an id (search / get managers) use this so an id
+    injected upstream by ``RequestIdMiddleware`` flows through to the
+    response, while direct / CLI callers still get a freshly minted id.
+    """
+    return get_request_id() or gen_request_id()

src/everos/core/lifespan/__init__.py

@@ -19,9 +19,11 @@
 from .base import LifespanProvider as LifespanProvider
 from .factory import build_lifespan as build_lifespan
 from .metrics_lifespan import MetricsLifespanProvider as MetricsLifespanProvider
+from .tracing_lifespan import TracingLifespanProvider as TracingLifespanProvider
 
 __all__ = [
     "LifespanProvider",
     "MetricsLifespanProvider",
+    "TracingLifespanProvider",
     "build_lifespan",
 ]

src/everos/core/lifespan/tracing_lifespan.py

@@ -0,0 +1,50 @@
+"""Tracing lifespan provider.
+
+Builds the OpenTelemetry ``TracerProvider`` at startup (from the
+``[observability]`` settings) and flushes + tears it down at shutdown.
+Chassis-level (backend-agnostic), so it lives here alongside
+``MetricsLifespanProvider`` rather than under the API entrypoint.
+
+Registered with a low ``order`` so the tracer is live before other
+providers start and can themselves be traced.
+"""
+
+from __future__ import annotations
+
+from fastapi import FastAPI
+
+from everos.config import load_settings
+from everos.core.observability.logging import get_logger
+from everos.core.observability.tracing import (
+    init_score_sink,
+    init_tracing,
+    shutdown_score_sink,
+    shutdown_tracing,
+)
+
+from .base import LifespanProvider
+
+logger = get_logger(__name__)
+
+
+class TracingLifespanProvider(LifespanProvider):
+    """Manages the OTel tracer provider + recall-score sink over the app life."""
+
+    def __init__(self, order: int = 1) -> None:
+        super().__init__(name="tracing", order=order)
+
+    async def startup(self, app: FastAPI) -> bool:
+        """Install the tracer provider + recall-score sink when configured.
+
+        Returns True if tracing was enabled and a provider installed.
+        """
+        settings = load_settings().observability
+        enabled = init_tracing(settings)
+        scores = init_score_sink(settings)
+        logger.info("tracing_lifespan_startup", enabled=enabled, scores=scores)
+        return enabled
+
+    async def shutdown(self, app: FastAPI) -> None:
+        await shutdown_score_sink()
+        shutdown_tracing()
+        logger.info("tracing_lifespan_shutdown")

src/everos/core/middleware/__init__.py

@@ -18,6 +18,7 @@
 from .cors import DEFAULT_CORS_ORIGINS as DEFAULT_CORS_ORIGINS
 from .profile import ProfileMiddleware as ProfileMiddleware
 from .prometheus import PrometheusMiddleware as PrometheusMiddleware
+from .request_id import RequestIdMiddleware as RequestIdMiddleware
 
 __all__ = [
     "DEFAULT_CORS_ALLOW_CREDENTIALS",
@@ -26,4 +27,5 @@
     "DEFAULT_CORS_ORIGINS",
     "ProfileMiddleware",
     "PrometheusMiddleware",
+    "RequestIdMiddleware",
 ]