From b50181f998179e573bab72d14d430fc3fc7f8905 Mon Sep 17 00:00:00 2001 From: Barrett Pyke Date: Fri, 18 Sep 2026 11:05:21 -0500 Subject: [PATCH 1/2] send x-bt-org-name header in make_scorer --- py/src/braintrust/devserver/server.py | 2 ++ py/src/braintrust/devserver/test_server.py | 30 ++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 py/src/braintrust/devserver/test_server.py diff --git a/py/src/braintrust/devserver/server.py b/py/src/braintrust/devserver/server.py index 3c4b612a..af291444 100644 --- a/py/src/braintrust/devserver/server.py +++ b/py/src/braintrust/devserver/server.py @@ -371,6 +371,8 @@ def scorer_fn(input, output, expected, metadata): "strict": True, } headers = {"Accept": "application/json", "Content-Type": "application/json"} + if state.org_name: + headers["x-bt-org-name"] = state.org_name if project_id: headers["x-bt-project-id"] = project_id result = state.proxy_conn().post("function/invoke", data=bt_dumps(request).encode("utf-8"), headers=headers) diff --git a/py/src/braintrust/devserver/test_server.py b/py/src/braintrust/devserver/test_server.py new file mode 100644 index 00000000..b983e47d --- /dev/null +++ b/py/src/braintrust/devserver/test_server.py @@ -0,0 +1,30 @@ +from unittest.mock import MagicMock + +import pytest +from braintrust.test_helpers import has_devserver_installed + + +def test_ui_scorer_forwards_org_name_to_function_invoke(): + if not has_devserver_installed(): + pytest.skip("Devserver dependencies not installed (requires .[cli])") + + from braintrust.devserver.server import make_scorer + + response = MagicMock() + response.json.return_value = {"score": 1} + connection = MagicMock() + connection.post.return_value = response + state = MagicMock() + state.org_name = "test-org" + state.proxy_conn.return_value = connection + state.current_span.get.return_value.export.return_value = {} + + scorer = make_scorer(state, "ui-scorer", {"function_id": "function-id"}, "project-id") + + assert scorer("input", "output", "expected", {}) == {"score": 1} + assert connection.post.call_args.args == ("function/invoke",) + assert connection.post.call_args.kwargs["headers"] == { + "Accept": "application/json", + "x-bt-org-name": "test-org", + "x-bt-project-id": "project-id", + } From c911b2b11fd74145928a4fea51025769a4c8a2e7 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 22 Sep 2026 10:53:09 -0400 Subject: [PATCH 2/2] more robust test --- py/src/braintrust/devserver/test_server.py | 60 ++++++++++++++++------ 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/py/src/braintrust/devserver/test_server.py b/py/src/braintrust/devserver/test_server.py index b983e47d..b4a7e8b7 100644 --- a/py/src/braintrust/devserver/test_server.py +++ b/py/src/braintrust/devserver/test_server.py @@ -1,30 +1,58 @@ -from unittest.mock import MagicMock +import http.server +import threading +from collections.abc import Iterator +from typing import Any import pytest +from braintrust.logger import BraintrustState from braintrust.test_helpers import has_devserver_installed -def test_ui_scorer_forwards_org_name_to_function_invoke(): +@pytest.fixture +def function_server() -> Iterator[tuple[str, list[str | None]]]: + org_names = [] + + class FunctionHandler(http.server.BaseHTTPRequestHandler): + def log_message(self, format: str, *args: Any) -> None: + pass + + def do_POST(self) -> None: + content_length = int(self.headers.get("Content-Length", "0")) + self.rfile.read(content_length) + org_names.append(self.headers.get("x-bt-org-name")) + + body = b'{"score": 1}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), FunctionHandler) + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + + try: + host, port = server.server_address + yield f"http://{host}:{port}", org_names + finally: + server.shutdown() + server.server_close() + thread.join() + + +def test_ui_scorer_forwards_org_name_to_function_invoke(function_server): if not has_devserver_installed(): pytest.skip("Devserver dependencies not installed (requires .[cli])") from braintrust.devserver.server import make_scorer - response = MagicMock() - response.json.return_value = {"score": 1} - connection = MagicMock() - connection.post.return_value = response - state = MagicMock() + base_url, org_names = function_server + state = BraintrustState() state.org_name = "test-org" - state.proxy_conn.return_value = connection - state.current_span.get.return_value.export.return_value = {} + state.proxy_url = base_url scorer = make_scorer(state, "ui-scorer", {"function_id": "function-id"}, "project-id") + scorer("input", "output", "expected", {}) - assert scorer("input", "output", "expected", {}) == {"score": 1} - assert connection.post.call_args.args == ("function/invoke",) - assert connection.post.call_args.kwargs["headers"] == { - "Accept": "application/json", - "x-bt-org-name": "test-org", - "x-bt-project-id": "project-id", - } + assert org_names == ["test-org"]