Skip to content

Harden currency response parsing #52

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wilsonfreitas merged 1 commit into from
Jun 14, 2026
Merged

Harden currency response parsing #52

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 58 additions & 25 deletions bcb/currency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
import httpx
import numpy as np
import pandas as pd
from lxml import html
from lxml import etree, html

from bcb.http import (
get_async_client,
Expand Down Expand Up @@ -309,6 +309,48 @@ def _raise_no_valid_currency_symbols(symbols: List[str]) -> NoReturn:
raise CurrencyNotFoundError(f"No valid currency symbols found: {requested}")


def _is_html_response(response: httpx.Response) -> bool:
content_type = response.headers.get("Content-Type", "").lower()
if content_type.startswith("text/html"):
return True
body = response.content.lstrip().lower()
return body.startswith((b"<!doctype html", b"<html", b"<body", b"<div"))


def _clean_currency_error_message(message: str) -> str:
message = re.sub(r"^\W+", "", message)
message = re.sub(r"\W+$", "", message)
message = re.sub(r"\s+", " ", message)
return message.strip()


def _extract_currency_html_error(content: bytes) -> str | None:
try:
doc = html.parse(BytesIO(content)).getroot()
except (etree.ParserError, ValueError):
return None
xpath = "//div[@class='msgErro']"
for element in doc.xpath(xpath):
message = _clean_currency_error_message(str(element.text_content()))
if message:
return message
return None


def _raise_currency_html_error(response: httpx.Response, symbol: str) -> NoReturn:
message = _extract_currency_html_error(response.content)
if message:
raise BCBAPIError(
f"BCB API returned error for {symbol}: {message}",
status_code=400,
)
raise BCBAPIError(
f"BCB API returned an HTML response for {symbol} "
"without a recognized BCB error message",
status_code=400,
)


def _fetch_symbol_response(
symbol: str, start_date: DateInput, end_date: DateInput
) -> "httpx.Response":
Expand Down Expand Up @@ -348,16 +390,9 @@ def _fetch_symbol_response(
f"Currency data response: status={res.status_code}, length={len(res.content)}"
)

# Handle HTML error response (e.g., no data for date range)
if res.headers.get("Content-Type", "").startswith("text/html"):
doc = html.parse(BytesIO(res.content)).getroot()
xpath = "//div[@class='msgErro']"
elm = doc.xpath(xpath)[0]
x = elm.text
x = re.sub(r"^\W+", "", x)
x = re.sub(r"\W+$", "", x)
msg = f"BCB API returned error: {x} - {symbol}"
raise BCBAPIError(msg, status_code=400)
# Handle HTML error responses (e.g., no data for date range).
if _is_html_response(res):
_raise_currency_html_error(res, symbol)

raise_for_status(
res,
Expand Down Expand Up @@ -386,7 +421,12 @@ def _validate_currency_csv(csv_text: str) -> pd.DataFrame:
BCBAPIError
If CSV format is invalid (wrong column count)
"""
df = pd.read_csv(StringIO(csv_text), delimiter=";", header=None, dtype=str)
if not csv_text.strip():
raise BCBAPIError("Currency CSV response is empty", status_code=400)
try:
df = pd.read_csv(StringIO(csv_text), delimiter=";", header=None, dtype=str)
except (pd.errors.EmptyDataError, pd.errors.ParserError) as e:
raise BCBAPIError(f"Failed to parse currency CSV: {e}", status_code=400) from e

# Validate column count
if len(df.columns) != 8:
Expand Down Expand Up @@ -420,10 +460,10 @@ def _parse_currency_dates(df: pd.DataFrame) -> pd.DataFrame:
"""
try:
df["Date"] = pd.to_datetime(df["Date"], format="%d%m%Y")
except ValueError as e:
except (TypeError, ValueError) as e:
raise BCBAPIError(
f"Failed to parse currency date column: {str(e)}", status_code=400
)
) from e
return df


Expand All @@ -448,10 +488,10 @@ def _parse_currency_types(df: pd.DataFrame) -> pd.DataFrame:
try:
df["bid"] = df["bid"].str.replace(",", ".").astype(np.float64)
df["ask"] = df["ask"].str.replace(",", ".").astype(np.float64)
except (ValueError, TypeError) as e:
except (TypeError, ValueError) as e:
raise BCBAPIError(
f"Failed to parse currency numeric columns: {str(e)}", status_code=400
)
) from e
return df


Expand Down Expand Up @@ -768,15 +808,8 @@ async def _async_fetch_symbol_response(
except httpx.HTTPError as ex:
raise_for_request_error(ex, context=f"Currency data for {symbol}")

if res.headers.get("Content-Type", "").startswith("text/html"):
doc = html.parse(BytesIO(res.content)).getroot()
xpath = "//div[@class='msgErro']"
elm = doc.xpath(xpath)[0]
x = elm.text
x = re.sub(r"^\W+", "", x)
x = re.sub(r"\W+$", "", x)
msg = f"BCB API returned error: {x} - {symbol}"
raise BCBAPIError(msg, status_code=400)
if _is_html_response(res):
_raise_currency_html_error(res, symbol)

raise_for_status(
res,
Expand Down
33 changes: 32 additions & 1 deletion tests/test_async.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@

from bcb import currency, sgs
from bcb.odata.api import Expectativas
from bcb.exceptions import BCBRateLimitError, CurrencyNotFoundError, ODataError
from bcb.exceptions import (
BCBAPIError,
BCBRateLimitError,
CurrencyNotFoundError,
ODataError,
)
from tests.conftest import (
CURRENCY_ID_LIST_HTML,
CURRENCY_LIST_CSV,
Expand Down Expand Up @@ -203,6 +208,32 @@ async def test_async_get_all_invalid_text_symbols_raise_clear_error(httpx_mock):
await currency.async_get(["ZAR", "ZZ1"], START, END, output="text")


async def test_async_get_symbol_unexpected_html_raises_bcb_error(httpx_mock):
add_currency_base_mocks(httpx_mock)
httpx_mock.add_response(
url=PTAX_RATE_URL,
text="<html><body><p>temporary failure</p></body></html>",
status_code=200,
headers={},
)

with pytest.raises(BCBAPIError, match="HTML response.*USD.*recognized"):
await currency._async_get_symbol("USD", START, END)


async def test_async_get_symbol_empty_response_body_raises_bcb_error(httpx_mock):
add_currency_base_mocks(httpx_mock)
httpx_mock.add_response(
url=PTAX_RATE_URL,
text="",
status_code=200,
headers={"Content-Type": "text/csv"},
)

with pytest.raises(BCBAPIError, match="empty"):
await currency._async_get_symbol("USD", START, END)


# ---------------------------------------------------------------------------
# OData async tests
# ---------------------------------------------------------------------------
Expand Down
109 changes: 107 additions & 2 deletions tests/test_currency_negative.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,111 @@ def test_fetch_symbol_429_rate_limit_raises(httpx_mock):
# ---------------------------------------------------------------------------


def test_fetch_symbol_html_error_extracts_message(httpx_mock):
"""HTML error pages with the expected element raise BCBAPIError."""
from tests.conftest import CURRENCY_ID_LIST_HTML, CURRENCY_LIST_CSV

httpx_mock.add_response(
url=PTAX_ID_LIST_URL,
content=CURRENCY_ID_LIST_HTML,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_CSV_DOWNLOAD_URL,
text=CURRENCY_LIST_CSV,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_RATE_URL,
text="<html><body><div class='msgErro'> No data available </div></body></html>",
status_code=200,
headers={"Content-Type": "text/html"},
)

with pytest.raises(BCBAPIError, match="No data available"):
currency._fetch_symbol_response("USD", START, END)


def test_fetch_symbol_unexpected_html_without_content_type_raises(httpx_mock):
"""HTML bodies without Content-Type or msgErro still raise BCBAPIError."""
from tests.conftest import CURRENCY_ID_LIST_HTML, CURRENCY_LIST_CSV

httpx_mock.add_response(
url=PTAX_ID_LIST_URL,
content=CURRENCY_ID_LIST_HTML,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_CSV_DOWNLOAD_URL,
text=CURRENCY_LIST_CSV,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_RATE_URL,
text="<html><body><p>temporary failure</p></body></html>",
status_code=200,
headers={},
)

with pytest.raises(BCBAPIError, match="HTML response.*USD.*recognized"):
currency._fetch_symbol_response("USD", START, END)


def test_get_symbol_missing_content_type_valid_csv_parses(httpx_mock):
"""Missing Content-Type alone does not reject a valid CSV response."""
from tests.conftest import (
CURRENCY_ID_LIST_HTML,
CURRENCY_LIST_CSV,
CURRENCY_RATE_CSV,
)

httpx_mock.add_response(
url=PTAX_ID_LIST_URL,
content=CURRENCY_ID_LIST_HTML,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_CSV_DOWNLOAD_URL,
text=CURRENCY_LIST_CSV,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_RATE_URL,
text=CURRENCY_RATE_CSV,
status_code=200,
headers={},
)

df = currency._get_symbol("USD", START, END)

assert ("USD", "bid") in df.columns


def test_get_symbol_empty_response_body_raises(httpx_mock):
"""Empty CSV responses raise BCBAPIError."""
from tests.conftest import CURRENCY_ID_LIST_HTML, CURRENCY_LIST_CSV

httpx_mock.add_response(
url=PTAX_ID_LIST_URL,
content=CURRENCY_ID_LIST_HTML,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_CSV_DOWNLOAD_URL,
text=CURRENCY_LIST_CSV,
status_code=200,
)
httpx_mock.add_response(
url=PTAX_RATE_URL,
text="",
status_code=200,
headers={"Content-Type": "text/csv"},
)

with pytest.raises(BCBAPIError, match="empty"):
currency._get_symbol("USD", START, END)


def test_get_symbol_malformed_csv_wrong_column_count_raises(httpx_mock):
"""Test that CSV with wrong column count raises BCBAPIError."""
from tests.conftest import CURRENCY_ID_LIST_HTML, CURRENCY_LIST_CSV
Expand Down Expand Up @@ -189,7 +294,7 @@ def test_get_symbol_malformed_csv_invalid_date_format_raises(httpx_mock):
text=malformed_csv,
status_code=200,
)
with pytest.raises(BCBAPIError):
with pytest.raises(BCBAPIError, match="date column"):
currency._get_symbol("USD", START, END)


Expand All @@ -214,7 +319,7 @@ def test_get_symbol_malformed_csv_invalid_numeric_conversion_raises(httpx_mock):
text=malformed_csv,
status_code=200,
)
with pytest.raises(BCBAPIError):
with pytest.raises(BCBAPIError, match="numeric columns"):
currency._get_symbol("USD", START, END)


Expand Down
Loading