Skip to content

Commit 316ed39

Browse files
AbhiPrasadclaude
andcommitted
fix(dspy): keep LiteLLM's 3MB price map out of the cassette
CI failed test_dspy(latest) on Windows across 3.11-3.14. Two separate problems, both from the LiteLLM 1.100.1 -> 1.102.0 bump. 1.102.0 fetches model_prices_and_context_window.json from GitHub on import. My earlier re-record captured it, which both bloated the cassette by ~3MB and displaced the tiktoken cl100k_base download that CI needs and my machine (warm tiktoken cache) never makes. Setting LITELLM_LOCAL_MODEL_COST_MAP keeps the fetch off the wire entirely, so the cassette goes back to being about the traffic under test -- it is now byte-identical to main again. The env var is scoped to the DSPy session on purpose: an older LiteLLM's bundled map does not know newer model names, and test_litellm(1.74.0) fails provider resolution for gpt-image-1-mini under it. Also make patcher module resolution consult sys.modules before importlib. Separately, test_mistral(latest) failed on ubuntu/3.10 with `KeyError: <thread id>` raised from importlib._bootstrap while resolve_root() imported mistralai.client.chat -- CPython 3.10's re-entrancy bookkeeping tripping over our own lock traffic. We call import_module for every patcher on every setup(), almost always for a module that is already imported, so the sys.modules fast path removes nearly all of it. That failure does not reproduce locally (3/3 green on 3.10), so this is a mitigation rather than a proven cure; it is a worthwhile change on its own merits either way. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1 parent 9b4868c commit 316ed39

4 files changed

Lines changed: 200811 additions & 35372 deletions

File tree

py/noxfile.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,6 +363,15 @@ def test_openai_agents(session, version):
363363

364364
LITELLM_VERSIONS = _get_matrix_versions("litellm")
365365

366+
# LiteLLM >= 1.102.0 fetches model_prices_and_context_window.json from GitHub on
367+
# import. Recording that lands a ~3MB blob in the cassette, so force the bundled
368+
# copy instead and keep the fetch off the wire.
369+
#
370+
# Only safe where the tests stick to chat completions: the bundled map of an
371+
# older LiteLLM does not know newer model names, and test_litellm(1.74.0) fails
372+
# provider resolution for gpt-image-1-mini under it. Scoped to DSPy for now.
373+
_LITELLM_LOCAL_COST_MAP = {"LITELLM_LOCAL_MODEL_COST_MAP": "True"}
374+
366375

367376
@nox.session()
368377
@nox.parametrize("version", LITELLM_VERSIONS, ids=LITELLM_VERSIONS)
@@ -597,7 +606,7 @@ def test_dspy(session, version):
597606
# dependency resolution does not select that incompatible release.
598607
_install_matrix_dep(session, "litellm", LATEST)
599608
_install_matrix_dep(session, "dspy", version)
600-
_run_tests(session, f"{INTEGRATION_DIR}/dspy/test_dspy.py", version=version)
609+
_run_tests(session, f"{INTEGRATION_DIR}/dspy/test_dspy.py", version=version, env=_LITELLM_LOCAL_COST_MAP)
601610

602611

603612
CREWAI_VERSIONS = _get_matrix_versions("crewai")

py/src/braintrust/integrations/base.py

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -93,10 +93,7 @@ def resolve_root(cls, module: Any | None, version: str | None, *, target: Any |
9393
if target is not None:
9494
return target
9595
if cls.target_module is not None:
96-
try:
97-
return importlib.import_module(cls.target_module)
98-
except ImportError:
99-
return None
96+
return _import_optional_module(cls.target_module)
10097
return module
10198

10299
@classmethod
@@ -147,10 +144,7 @@ def resolve_scan_root(cls, module: Any | None, version: str | None, *, target: A
147144
if target is not None:
148145
return target
149146
if cls.target_module is not None:
150-
try:
151-
return importlib.import_module(cls.target_module)
152-
except ImportError:
153-
return None
147+
return _import_optional_module(cls.target_module)
154148
return module
155149

156150
@classmethod
@@ -274,10 +268,7 @@ def resolve_root(cls, module: Any | None, version: str | None, *, target: Any |
274268
if target is not None:
275269
return target
276270
if cls.target_module is not None:
277-
try:
278-
return importlib.import_module(cls.target_module)
279-
except ImportError:
280-
return None
271+
return _import_optional_module(cls.target_module)
281272
return module
282273

283274
@classmethod
@@ -636,13 +627,30 @@ def detect_version(cls, module: Any) -> str | None:
636627
return detect_module_version(module, cls.import_names)
637628

638629

630+
def _import_optional_module(name: str) -> Any | None:
631+
"""Return the named module, or ``None`` when it cannot be imported.
632+
633+
``sys.modules`` is consulted first so an already-imported module never
634+
reacquires the import lock. Patcher resolution calls this for every
635+
patcher on every ``setup()``, and on CPython 3.10 that lock traffic can
636+
trip the interpreter's own re-entrancy bookkeeping, surfacing as
637+
``KeyError: <thread id>`` from ``importlib._bootstrap``.
638+
"""
639+
module = sys.modules.get(name)
640+
if module is not None:
641+
return module
642+
try:
643+
return importlib.import_module(name)
644+
except ImportError:
645+
return None
646+
647+
639648
def _import_first_available(import_names: Iterable[str]) -> Any | None:
640649
"""Import and return the first available module from the given names."""
641650
for import_name in import_names:
642-
try:
643-
return importlib.import_module(import_name)
644-
except ImportError:
645-
continue
651+
module = _import_optional_module(import_name)
652+
if module is not None:
653+
return module
646654
return None
647655

648656

0 commit comments

Comments
 (0)