test: prune redundant integration tests and add unused-cassette checker

AGENTS.md

@@ -244,6 +244,17 @@ Stale cassette detection:
 - To auto-delete stale dirs: `cd py && python scripts/check-stale-cassettes.py --clean`
 - When adding a new integration with versioned cassettes, add an entry to `[tool.braintrust.cassette-dirs]`.
 
+Unused cassette detection:
+
+- `check-stale-cassettes.py` only checks whole version directories. Individual files inside a valid directory go stale when a test is deleted or renamed, or when a test always skips at that version.
+- `py/scripts/check-unused-cassettes.py` finds those files. It runs every nox session that reads an integration's cassettes, across all matrix versions in replay-only mode, and reports files no test opened.
+- Reads are recorded by an audit hook in `py/src/braintrust/_test_cassette_usage.py`, active only when `BRAINTRUST_CASSETTE_USAGE_DIR` is set. It covers every loader (pytest-recording, Claude Agent SDK transport, gRPC recordings, btx specs) and auto-instrument subprocesses.
+- Check specific integrations: `cd py && make check-unused-cassettes INTEGRATIONS="openai anthropic"` (omit `INTEGRATIONS` to check all; it is slow).
+- Add `--clean` to delete what it finds: `cd py && python scripts/check-unused-cassettes.py run openai --clean`.
+- An integration is skipped, not reported, if any of its sessions fails or is skipped (e.g. sessions that skip on the current Python version), since its reads would be incomplete.
+- Unread files in a cassette version directory whose session skipped any test (e.g. platform-only tests) are listed with the skip reasons for manual review and never deleted by `--clean`, since the skipped test may read them elsewhere.
+- After deleting or renaming a cassette-backed test, run it for that integration.
+
 ## Benchmarks
 
 If you touch a hot path such as serialization, deep-copy, span creation, or logging, consider benchmarks.

py/Makefile

@@ -1,6 +1,6 @@
 PYTHON ?= python
 
-.PHONY: lint pylint test test-wheel _template-version clean fixup build verify-build verify help install-build-deps install-dev test-core check-stale-cassettes sync-pytest-pin fetch-openapi-spec generate-api-client check-api-client-codegen test-api-codegen _check-git-clean bench bench-compare
+.PHONY: lint pylint test test-wheel _template-version clean fixup build verify-build verify help install-build-deps install-dev test-core check-stale-cassettes check-unused-cassettes sync-pytest-pin fetch-openapi-spec generate-api-client check-api-client-codegen test-api-codegen _check-git-clean bench bench-compare
 
 clean:
 	rm -rf build dist
@@ -33,6 +33,9 @@ test-core:
 check-stale-cassettes:
 	$(PYTHON) scripts/check-stale-cassettes.py
 
+check-unused-cassettes:
+	$(PYTHON) scripts/check-unused-cassettes.py run $(or $(INTEGRATIONS),--all)
+
 sync-pytest-pin:
 	$(PYTHON) scripts/sync-pytest-pin.py
 
@@ -92,6 +95,7 @@ help:
 	@echo "  bench-compare       - Compare two benchmark results (BENCH_BASE=... BENCH_NEW=...)"
 	@echo "  build               - Build Python package"
 	@echo "  check-stale-cassettes - Detect orphaned cassette version directories"
+	@echo "  check-unused-cassettes - Find cassette files no test loads (INTEGRATIONS=\"openai anthropic\"; default all)"
 	@echo "  fetch-openapi-spec  - Fetch the hash-verified pinned OpenAPI spec"
 	@echo "  generate-api-client - Generate the REST API client and public reference from the pinned spec"
 	@echo "  check-api-client-codegen - Check the committed REST API client and public reference for drift"

py/noxfile.py

@@ -605,7 +605,7 @@ def test_dspy(session, version):
         # and Python 3.14 wheels. Preinstall our portable matrix pin so DSPy's
         # dependency resolution does not select that incompatible release.
         _install_matrix_dep(session, "litellm", LATEST)
-    _install_matrix_dep(session, "dspy", version)
+    _install_matrix_dep(session, "dspy", version, "test-sqlalchemy-2-0")
     _run_tests(session, f"{INTEGRATION_DIR}/dspy/test_dspy.py", version=version, env=_LITELLM_LOCAL_COST_MAP)
 
 
@@ -632,7 +632,7 @@ def test_crewai(session, version):
 @nox.parametrize("version", GOOGLE_ADK_VERSIONS, ids=GOOGLE_ADK_VERSIONS)
 def test_google_adk(session, version):
     _install_test_deps(session)
-    _install_matrix_dep(session, "google-adk", version)
+    _install_matrix_dep(session, "google-adk", version, "test-sqlalchemy-2-0")
     _run_tests(session, f"{INTEGRATION_DIR}/adk/test_adk.py", version=version)
 
 
@@ -672,7 +672,7 @@ def test_deepagents(session, version):
 def test_llamaindex(session, version):
     _install_test_deps(session)
     _install_group_locked(session, "test-llamaindex")
-    _install_matrix_dep(session, "llama-index-core", version)
+    _install_matrix_dep(session, "llama-index-core", version, "test-sqlalchemy-2-0")
     # These packages are tightly version-coupled to llama-index-core, so we
     # install them unpinned and let pip resolve compatible versions.
     session.install("llama-index-llms-openai", "llama-index-embeddings-openai", silent=SILENT_INSTALLS)

py/pyproject.toml

@@ -152,6 +152,12 @@ test-mcp-v1 = [
     "mcp==1.28.1",
 ]
 
+# SQLAlchemy 2.1.0 ships only an sdist whose pyproject declares a duplicate
+# normalized extra (`mssql-pymssql`), which uv refuses to build.
+test-sqlalchemy-2-0 = [
+    "sqlalchemy<2.1",
+]
+
 test-pydantic-ai-otel-events = [
     "opentelemetry-api==1.39.1",
     # genai-prices 0.1 returns fields that older Pydantic AI RequestUsage models cannot accept.