feat(observability): enhance AstraDB instrumentation and update dependencies

- Added runtime patching for AstraDBCollection methods to instrument database operations with OpenTelemetry spans and Prometheus histograms.
- Updated observability settings in `config.py` to include new OTLP protocol options and headers.
- Enhanced logging configuration to support JSON formatting and improved error handling for logging setup.
- Updated dependencies in `pyproject.toml` and `poetry.lock` for FastAPI, cryptography, and other packages to their latest versions.
- Introduced a GitHub workflow for validating JSON files in the Grafana dashboard directory.

Author: pmcfadin

.github/workflows/dashboard-lint.yml

@@ -0,0 +1,23 @@
+name: Grafana JSON lint
+
+on:
+  push:
+    paths:
+      - 'docs/grafana/*.json'
+  pull_request:
+    paths:
+      - 'docs/grafana/*.json'
+
+jobs:
+  json-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Validate JSON
+        run: |
+          sudo apt-get update -y
+          sudo apt-get install -y jq
+          for file in docs/grafana/*.json; do
+            echo "Linting $file";
+            jq type "$file" > /dev/null;
+          done 

app/core/config.py

@@ -156,7 +156,10 @@ def _sanitize_bool_tokens(cls, data):  # type: ignore[return-value]
     # initialised.  This is useful for local development or extremely
     # resource-constrained environments where you don't want the overhead of
     # telemetry.
-    OBSERVABILITY_ENABLED: bool = Field(default=True, description="Globally enable/disable all extra observability (metrics/traces/log shipping).")
+    OBSERVABILITY_ENABLED: bool = Field(
+        default=True,
+        description="Globally enable/disable all extra observability (metrics/traces/log shipping).",
+    )
 
     # --- OpenTelemetry ---------------------------------------------------
 
@@ -177,12 +180,21 @@ def _sanitize_bool_tokens(cls, data):  # type: ignore[return-value]
     # pull metrics if desired.
     OTEL_METRICS_ENABLED: bool = False
 
+    # Protocol for OTLP export – "grpc" (default) or "http". Allows integration with collectors that only expose the HTTP/JSON OTLP endpoint (4318).
+    OTEL_EXPORTER_OTLP_PROTOCOL: str = Field(default="grpc")
+
+    # Optional additional headers to send along OTLP requests (for auth tokens, etc.).
+    # Provide as comma-separated key=value list, e.g. "mcp-token=abcd123,env=dev".
+    OTEL_EXPORTER_OTLP_HEADERS: str | None = Field(default=None)
+
     # Sample ratio (0.0-1.0) for traces – 1.0 = always.
     OTEL_TRACES_SAMPLER_RATIO: float = Field(default=1.0, ge=0.0, le=1.0)
 
     # --- Centralised logging (Loki) --------------------------------------
 
-    LOKI_ENABLED: bool = Field(default=False, description="Enable structured log shipping to Loki.")
+    LOKI_ENABLED: bool = Field(
+        default=False, description="Enable structured log shipping to Loki."
+    )
     LOKI_ENDPOINT: str | None = Field(
         default=None,
         description="Loki push API endpoint, e.g. http://loki:3100/loki/api/v1/push.",

app/utils/db_instrumentation.py

@@ -0,0 +1,65 @@
+"""Runtime patching helpers to instrument AstraDBCollection methods.
+
+Call `instrument_astra_collection()` early during application startup (done in
+`app.utils.observability`) and every `insert_one` / `update_one` will be
+surrounded by an OpenTelemetry span and Prometheus histogram sample.  This
+provides visibility into **all** Data-API mutations without having to wrap
+each individual call site.
+"""
+from __future__ import annotations
+
+import time
+from typing import Any, Awaitable
+
+from opentelemetry import trace
+
+from app.metrics import ASTRA_DB_QUERY_DURATION_SECONDS
+
+_tracer = trace.get_tracer(__name__)
+
+
+async def _observe(op: str, coro: Awaitable[Any]):  # noqa: D401
+    """Await *coro* while recording span + histogram for DB *op*."""
+
+    start = time.perf_counter()
+    with _tracer.start_as_current_span(f"astra.{op}") as span:
+        try:
+            result = await coro
+            return result
+        finally:
+            duration = time.perf_counter() - start
+            ASTRA_DB_QUERY_DURATION_SECONDS.labels(operation=op).observe(duration)
+            span.set_attribute("duration_ms", int(duration * 1000))
+
+
+def instrument_astra_collection() -> None:  # noqa: D401
+    """Monkey-patch AstraDBCollection once per process."""
+
+    try:
+        from app.db.astra_client import AstraDBCollection  # type: ignore
+    except Exception:
+        return  # Library not available in unit-test mode
+
+    if getattr(AstraDBCollection, "_kv_instrumented", False):
+        return  # Already patched
+
+    # ---------------------------- insert_one ----------------------------
+    if hasattr(AstraDBCollection, "insert_one"):
+        _orig_insert = AstraDBCollection.insert_one
+
+        async def _insert_wrap(self, *args, **kwargs):  # type: ignore
+            return await _observe("insert", _orig_insert(self, *args, **kwargs))
+
+        AstraDBCollection.insert_one = _insert_wrap  # type: ignore[attr-defined]
+
+    # ---------------------------- update_one ----------------------------
+    if hasattr(AstraDBCollection, "update_one"):
+        _orig_update = AstraDBCollection.update_one
+
+        async def _update_wrap(self, *args, **kwargs):  # type: ignore
+            return await _observe("update", _orig_update(self, *args, **kwargs))
+
+        AstraDBCollection.update_one = _update_wrap  # type: ignore[attr-defined]
+
+    # Mark as patched
+    AstraDBCollection._kv_instrumented = True  # type: ignore[attr-defined]