diff --git a/bcb/currency.py b/bcb/currency.py index 9b51efd..79889e3 100644 --- a/bcb/currency.py +++ b/bcb/currency.py @@ -1,13 +1,7 @@ -import re -import warnings -from datetime import date, timedelta -from io import BytesIO, StringIO from typing import Dict, List, Literal, Optional, Union, overload import httpx -import numpy as np import pandas as pd -from lxml import html from .exceptions import BCBAPIError, CurrencyNotFoundError from .utils import Date, DateInput @@ -16,16 +10,7 @@ O módulo :py:mod:`bcb.currency` tem como objetivo fazer consultas no site do conversor de moedas do BCB. """ - -def _currency_url(currency_id: int, start_date: DateInput, end_date: DateInput) -> str: - start_date = Date(start_date) - end_date = Date(end_date) - return ( - f"https://ptax.bcb.gov.br/ptax_internet/consultaBoletim.do?" - f"method=gerarCSVFechamentoMoedaNoPeriodo&" - f"ChkMoeda={currency_id}&DATAINI={start_date.date:%d/%m/%Y}&DATAFIM={end_date.date:%d/%m/%Y}" - ) - +_PTAX_BASE_URL = "https://olinda.bcb.gov.br/olinda/servico/PTAX/versao/v1/odata" _CACHE: dict[str, pd.DataFrame] = dict() @@ -33,51 +18,15 @@ def _currency_url(currency_id: int, start_date: DateInput, end_date: DateInput) def clear_cache() -> None: """Clear the module-level session cache. - :func:`get` and :func:`get_currency_list` cache the currency ID list and - the full currency master table for the duration of the Python session so - that repeated calls do not make redundant HTTP requests. Call this - function to force a fresh fetch on the next request (useful in tests or - long-running scripts where the master data may have changed). + :func:`get` and :func:`get_currency_list` cache the currency list for the + duration of the Python session so that repeated calls do not make redundant + HTTP requests. Call this function to force a fresh fetch on the next + request (useful in tests or long-running scripts where the data may have + changed). """ _CACHE.clear() -def _currency_id_list() -> pd.DataFrame: - if _CACHE.get("TEMP_CURRENCY_ID_LIST") is not None: - return _CACHE.get("TEMP_CURRENCY_ID_LIST") - else: - url1 = ( - "https://ptax.bcb.gov.br/ptax_internet/consultaBoletim.do?" - "method=exibeFormularioConsultaBoletim" - ) - res = httpx.get(url1, follow_redirects=True) - if res.status_code != 200: - msg = f"BCB API Request error, status code = {res.status_code}" - raise BCBAPIError(msg, res.status_code) - - doc = html.parse(BytesIO(res.content)).getroot() - xpath = "//select[@name='ChkMoeda']/option" - x = [(elm.text, elm.get("value")) for elm in doc.xpath(xpath)] - df = pd.DataFrame(x, columns=["name", "id"]) - df["id"] = df["id"].astype("int32") - _CACHE["TEMP_CURRENCY_ID_LIST"] = df - return df - - -def _get_valid_currency_list(_date: date, n: int = 0) -> httpx.Response: - url2 = f"http://www4.bcb.gov.br/Download/fechamento/M{_date:%Y%m%d}.csv" - try: - res = httpx.get(url2, follow_redirects=True) - except httpx.ConnectError as ex: - if n >= 3: - raise ex - return _get_valid_currency_list(_date, n + 1) - if res.status_code == 200: - return res - else: - return _get_valid_currency_list(_date - timedelta(1), 0) - - def get_currency_list() -> pd.DataFrame: """ Listagem com todas as moedas disponíveis na API e suas configurações de paridade. @@ -86,59 +35,55 @@ def get_currency_list() -> pd.DataFrame: ------- DataFrame : - Tabela com a listagem de moedas disponíveis. + Tabela com a listagem de moedas disponíveis (colunas: ``symbol``, + ``name``, ``type``). """ - if _CACHE.get("TEMP_FILE_CURRENCY_LIST") is not None: - return _CACHE.get("TEMP_FILE_CURRENCY_LIST") - else: - res = _get_valid_currency_list(date.today()) - df = pd.read_csv(StringIO(res.text), delimiter=";") - df.columns = [ - "code", - "name", - "symbol", - "country_code", - "country_name", - "type", - "exclusion_date", - ] - df = df.loc[~df["country_code"].isna()] - df["exclusion_date"] = pd.to_datetime(df["exclusion_date"], dayfirst=True) - df["country_code"] = df["country_code"].astype("int32") - df["code"] = df["code"].astype("int32") - df["symbol"] = df["symbol"].str.strip() - _CACHE["TEMP_FILE_CURRENCY_LIST"] = df - return df - - -def _get_currency_id(symbol: str) -> int: - id_list = _currency_id_list() + cached = _CACHE.get("TEMP_FILE_CURRENCY_LIST") + if cached is not None: + return cached + url = f"{_PTAX_BASE_URL}/Moedas?$format=json" + res = httpx.get(url, follow_redirects=True) + if res.status_code != 200: + msg = f"BCB API Request error, status code = {res.status_code}" + raise BCBAPIError(msg, res.status_code) + data = res.json() + df = pd.DataFrame(data["value"]) + df = df.rename( + columns={"simbolo": "symbol", "nomeFormatado": "name", "tipoMoeda": "type"} + ) + _CACHE["TEMP_FILE_CURRENCY_LIST"] = df + return df + + +def _validate_currency_symbol(symbol: str) -> None: all_currencies = get_currency_list() - x = pd.merge(id_list, all_currencies, on=["name"]) - matches = x.loc[x["symbol"] == symbol, "id"] - if matches.empty: + if symbol not in all_currencies["symbol"].values: raise CurrencyNotFoundError(f"Unknown currency symbol: {symbol}") - return int(matches.max()) + + +def _currency_url(symbol: str, start_date: DateInput, end_date: DateInput) -> str: + start_date = Date(start_date) + end_date = Date(end_date) + return ( + f"{_PTAX_BASE_URL}/CotacaoMoedaPeriodo(" + f"moeda=@moeda,dataInicial=@dataInicial,dataFinalCotacao=@dataFinalCotacao)?" + f"@moeda='{symbol}'&" + f"@dataInicial='{start_date.date:%m-%d-%Y}'&" + f"@dataFinalCotacao='{end_date.date:%m-%d-%Y}'&" + f"$format=json" + ) def _fetch_symbol_response( symbol: str, start_date: DateInput, end_date: DateInput ) -> Optional[httpx.Response]: try: - cid = _get_currency_id(symbol) + _validate_currency_symbol(symbol) except CurrencyNotFoundError: return None - url = _currency_url(cid, start_date, end_date) + url = _currency_url(symbol, start_date, end_date) res = httpx.get(url, follow_redirects=True) - if res.headers["Content-Type"].startswith("text/html"): - doc = html.parse(BytesIO(res.content)).getroot() - xpath = "//div[@class='msgErro']" - elm = doc.xpath(xpath)[0] - x = elm.text - x = re.sub(r"^\W+", "", x) - x = re.sub(r"\W+$", "", x) - msg = f"BCB API returned error: {x} - {symbol}" - warnings.warn(msg) + if res.status_code != 200: return None return res @@ -149,18 +94,17 @@ def _get_symbol( res = _fetch_symbol_response(symbol, start_date, end_date) if res is None: return None - columns = ["Date", "aa", "bb", "cc", "bid", "ask", "dd", "ee"] - df = pd.read_csv( - StringIO(res.text), delimiter=";", header=None, names=columns, dtype=str - ) - df = df.assign( - Date=lambda x: pd.to_datetime(x["Date"], format="%d%m%Y"), - bid=lambda x: x["bid"].str.replace(",", ".").astype(np.float64), - ask=lambda x: x["ask"].str.replace(",", ".").astype(np.float64), - ) - df1 = df.set_index("Date") + data = res.json() + if not data.get("value"): + return None + df = pd.DataFrame(data["value"]) + df = df[df["tipoBoletim"] == "Fechamento"].copy() + if df.empty: + return None + df["Date"] = pd.to_datetime(df["dataHoraCotacao"]).dt.normalize() + df = df.rename(columns={"cotacaoCompra": "bid", "cotacaoVenda": "ask"}) n = ["bid", "ask"] - df1 = df1[n] + df1 = df.set_index("Date")[n] tuples = list(zip([symbol] * len(n), n)) df1.columns = pd.MultiIndex.from_tuples(tuples) return df1 diff --git a/tests/conftest.py b/tests/conftest.py index 7596e14..f4471af 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +import json + import pytest from bcb import currency @@ -5,27 +7,51 @@ # Mock data constants # --------------------------------------------------------------------------- -CURRENCY_ID_LIST_HTML = b""" -
- -
-""" - -# First row is treated as header by pd.read_csv, then overwritten by df.columns = [...] -CURRENCY_LIST_CSV = ( - "Codigo;Nome;Simbolo;CodPais;NomePais;Tipo;DataExclusao\n" - "61;DOLLAR DOS EUA;USD;249;EUA;A;\n" +# OData /Moedas response +CURRENCY_LIST_JSON = json.dumps( + { + "value": [ + {"simbolo": "USD", "nomeFormatado": "DOLLAR DOS EUA", "tipoMoeda": "A"}, + ] + } ) -# 8 columns, no header, date format DDMMYYYY, comma as decimal separator -CURRENCY_RATE_CSV = ( - "01122020;0;0;0;5,0000;5,1000;0;0\n" - "02122020;0;0;0;5,0100;5,1100;0;0\n" - "03122020;0;0;0;5,0200;5,1200;0;0\n" - "04122020;0;0;0;5,0300;5,1300;0;0\n" - "07122020;0;0;0;5,0400;5,1400;0;0\n" +# OData /CotacaoMoedaPeriodo response — one "Fechamento" bulletin per trading day +CURRENCY_RATE_ODATA_JSON = json.dumps( + { + "value": [ + { + "cotacaoCompra": 5.0000, + "cotacaoVenda": 5.1000, + "dataHoraCotacao": "2020-12-01 13:03:38.273", + "tipoBoletim": "Fechamento", + }, + { + "cotacaoCompra": 5.0100, + "cotacaoVenda": 5.1100, + "dataHoraCotacao": "2020-12-02 13:03:38.273", + "tipoBoletim": "Fechamento", + }, + { + "cotacaoCompra": 5.0200, + "cotacaoVenda": 5.1200, + "dataHoraCotacao": "2020-12-03 13:03:38.273", + "tipoBoletim": "Fechamento", + }, + { + "cotacaoCompra": 5.0300, + "cotacaoVenda": 5.1300, + "dataHoraCotacao": "2020-12-04 13:03:38.273", + "tipoBoletim": "Fechamento", + }, + { + "cotacaoCompra": 5.0400, + "cotacaoVenda": 5.1400, + "dataHoraCotacao": "2020-12-07 13:03:38.273", + "tipoBoletim": "Fechamento", + }, + ] + } ) SGS_JSON_5 = ( diff --git a/tests/test_currency.py b/tests/test_currency.py index 429ecb6..eb923ee 100644 --- a/tests/test_currency.py +++ b/tests/test_currency.py @@ -7,68 +7,61 @@ from bcb import currency from bcb.exceptions import CurrencyNotFoundError from tests.conftest import ( - CURRENCY_ID_LIST_HTML, - CURRENCY_LIST_CSV, - CURRENCY_RATE_CSV, + CURRENCY_LIST_JSON, + CURRENCY_RATE_ODATA_JSON, ) START = datetime(2020, 12, 1) END = datetime(2020, 12, 7) -PTAX_ID_LIST_URL = re.compile(r".*exibeFormularioConsultaBoletim.*") -PTAX_CSV_DOWNLOAD_URL = re.compile(r".*www4\.bcb\.gov\.br.*\.csv") -PTAX_RATE_URL = re.compile(r".*gerarCSVFechamento.*") - - -def add_id_list_mock(httpx_mock): - httpx_mock.add_response( - url=PTAX_ID_LIST_URL, - content=CURRENCY_ID_LIST_HTML, - status_code=200, - ) +PTAX_MOEDAS_URL = re.compile(r".*olinda\.bcb\.gov\.br.*Moedas.*") +PTAX_RATE_ODATA_URL = re.compile(r".*olinda\.bcb\.gov\.br.*CotacaoMoedaPeriodo.*") def add_currency_list_mock(httpx_mock): httpx_mock.add_response( - url=PTAX_CSV_DOWNLOAD_URL, - text=CURRENCY_LIST_CSV, + url=PTAX_MOEDAS_URL, + text=CURRENCY_LIST_JSON, status_code=200, + headers={"Content-Type": "application/json"}, ) def add_rate_mock(httpx_mock): httpx_mock.add_response( - url=PTAX_RATE_URL, - text=CURRENCY_RATE_CSV, + url=PTAX_RATE_ODATA_URL, + text=CURRENCY_RATE_ODATA_JSON, status_code=200, - headers={"Content-Type": "text/csv"}, + headers={"Content-Type": "application/json"}, ) # --------------------------------------------------------------------------- -# _currency_id_list +# get_currency_list # --------------------------------------------------------------------------- -def test_currency_id_list(httpx_mock): - add_id_list_mock(httpx_mock) - df = currency._currency_id_list() +def test_get_currency_list(httpx_mock): + add_currency_list_mock(httpx_mock) + df = currency.get_currency_list() assert isinstance(df, pd.DataFrame) - assert list(df.columns) == ["name", "id"] - assert 61 in df["id"].values + assert "symbol" in df.columns + assert "name" in df.columns + assert "type" in df.columns + assert "USD" in df["symbol"].values -def test_currency_id_list_cached(httpx_mock): - add_id_list_mock(httpx_mock) - df1 = currency._currency_id_list() - df2 = currency._currency_id_list() # should use cache — no second HTTP call +def test_get_currency_list_cached(httpx_mock): + add_currency_list_mock(httpx_mock) + df1 = currency.get_currency_list() + df2 = currency.get_currency_list() # should use cache — no second HTTP call assert df1 is df2 def test_clear_cache(httpx_mock): # Populate the cache with one call - add_id_list_mock(httpx_mock) - currency._currency_id_list() + add_currency_list_mock(httpx_mock) + currency.get_currency_list() assert currency._CACHE # cache is non-empty # clear_cache() empties it @@ -76,41 +69,26 @@ def test_clear_cache(httpx_mock): assert not currency._CACHE # A subsequent call re-fetches and re-populates - add_id_list_mock(httpx_mock) - currency._currency_id_list() - assert currency._CACHE - - -# --------------------------------------------------------------------------- -# get_currency_list -# --------------------------------------------------------------------------- - - -def test_get_currency_list(httpx_mock): add_currency_list_mock(httpx_mock) - df = currency.get_currency_list() - assert isinstance(df, pd.DataFrame) - assert "symbol" in df.columns - assert "USD" in df["symbol"].values - assert df.loc[df["symbol"] == "USD", "code"].iloc[0] == 61 + currency.get_currency_list() + assert currency._CACHE # --------------------------------------------------------------------------- -# _get_currency_id +# _validate_currency_symbol # --------------------------------------------------------------------------- -def test_get_currency_id_found(httpx_mock): - add_id_list_mock(httpx_mock) +def test_validate_currency_symbol_found(httpx_mock): add_currency_list_mock(httpx_mock) - assert currency._get_currency_id("USD") == 61 + # Should not raise + currency._validate_currency_symbol("USD") -def test_get_currency_id_not_found(httpx_mock): - add_id_list_mock(httpx_mock) +def test_validate_currency_symbol_not_found(httpx_mock): add_currency_list_mock(httpx_mock) with pytest.raises(CurrencyNotFoundError, match="ZAR"): - currency._get_currency_id("ZAR") + currency._validate_currency_symbol("ZAR") # --------------------------------------------------------------------------- @@ -119,7 +97,6 @@ def test_get_currency_id_not_found(httpx_mock): def test_get_symbol_returns_dataframe(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) df = currency._get_symbol("USD", START, END) @@ -131,7 +108,6 @@ def test_get_symbol_returns_dataframe(httpx_mock): def test_get_symbol_unknown_currency_returns_none(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) result = currency._get_symbol("ZAR", START, END) assert result is None @@ -143,7 +119,6 @@ def test_get_symbol_unknown_currency_returns_none(httpx_mock): def test_currency_get_ask(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) df = currency.get("USD", START, END, side="ask") @@ -153,7 +128,6 @@ def test_currency_get_ask(httpx_mock): def test_currency_get_bid(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) df = currency.get("USD", START, END, side="bid") @@ -162,7 +136,6 @@ def test_currency_get_bid(httpx_mock): def test_currency_get_both_symbol_groupby(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) df = currency.get("USD", START, END, side="both", groupby="symbol") @@ -172,7 +145,6 @@ def test_currency_get_both_symbol_groupby(httpx_mock): def test_currency_get_both_side_groupby(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) df = currency.get("USD", START, END, side="both", groupby="side") @@ -182,7 +154,6 @@ def test_currency_get_both_side_groupby(httpx_mock): def test_currency_get_invalid_side(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) with pytest.raises(ValueError, match="Unknown side"): @@ -190,50 +161,46 @@ def test_currency_get_invalid_side(httpx_mock): def test_currency_get_unknown_symbol_raises(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) with pytest.raises(CurrencyNotFoundError): currency.get("ZAR", START, END) def test_currency_get_list_all_unknown_raises(httpx_mock): - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) with pytest.raises(CurrencyNotFoundError): currency.get(["ZAR", "ZZ1"], START, END) # --------------------------------------------------------------------------- -# output="text" — raw CSV string +# output="text" — raw JSON string # --------------------------------------------------------------------------- def test_currency_get_output_text_single_returns_string(httpx_mock): - """get('USD', output='text') returns the raw CSV string.""" - add_id_list_mock(httpx_mock) + """get('USD', output='text') returns the raw JSON string.""" add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) result = currency.get("USD", START, END, output="text") assert isinstance(result, str) - assert "01122020" in result + assert "2020-12-01" in result def test_currency_get_output_text_multi_returns_dict(httpx_mock): - """get(['USD', 'USD'], output='text') returns a dict mapping symbol → CSV.""" - add_id_list_mock(httpx_mock) + """get(['USD', 'USD'], output='text') returns a dict mapping symbol → JSON.""" add_currency_list_mock(httpx_mock) # Two calls for USD (same mock symbol, different entries in symbols list) httpx_mock.add_response( - url=PTAX_RATE_URL, - text=CURRENCY_RATE_CSV, + url=PTAX_RATE_ODATA_URL, + text=CURRENCY_RATE_ODATA_JSON, status_code=200, - headers={"Content-Type": "text/csv"}, + headers={"Content-Type": "application/json"}, ) httpx_mock.add_response( - url=PTAX_RATE_URL, - text=CURRENCY_RATE_CSV, + url=PTAX_RATE_ODATA_URL, + text=CURRENCY_RATE_ODATA_JSON, status_code=200, - headers={"Content-Type": "text/csv"}, + headers={"Content-Type": "application/json"}, ) result = currency.get(["USD", "USD"], START, END, output="text") assert isinstance(result, dict) @@ -243,7 +210,6 @@ def test_currency_get_output_text_multi_returns_dict(httpx_mock): def test_currency_get_output_text_unknown_raises(httpx_mock): """get('ZAR', output='text') raises CurrencyNotFoundError.""" - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) with pytest.raises(CurrencyNotFoundError): currency.get("ZAR", START, END, output="text") @@ -251,7 +217,6 @@ def test_currency_get_output_text_unknown_raises(httpx_mock): def test_currency_get_output_dataframe_is_default(httpx_mock): """Default output still returns DataFrame.""" - add_id_list_mock(httpx_mock) add_currency_list_mock(httpx_mock) add_rate_mock(httpx_mock) result = currency.get("USD", START, END)