From 7da5357e58e45179709e6bada05da0167b093555 Mon Sep 17 00:00:00 2001 From: Josh Terrill Date: Fri, 10 Apr 2026 16:20:42 -0700 Subject: [PATCH] added submission scores and pending model payouts. soft deprecated round model performances v2 --- numerapi/base_api.py | 144 ++++++++++++++++++++++++++++++++++++ tests/conftest.py | 63 ++++++++++++++++ tests/test_base_api.py | 154 +++++++++++++++++++++++++++++++++++++++ tests/test_numerapi.py | 4 + tests/test_signalsapi.py | 1 + 5 files changed, 366 insertions(+) diff --git a/numerapi/base_api.py b/numerapi/base_api.py index d6d49f2..03370ae 100644 --- a/numerapi/base_api.py +++ b/numerapi/base_api.py @@ -3,6 +3,7 @@ import datetime import logging import os +import warnings from io import BytesIO from typing import Dict, List, Tuple, Union @@ -841,9 +842,145 @@ def diagnostics(self, model_id: str, diagnostics_id: str | None = None) -> Dict: utils.replace(results, "updatedAt", utils.parse_datetime_string) return results + def submission_scores( + self, + model_id: str, + display_name: str | None = None, + version: str | None = None, + day: int | None = None, + resolved: bool | None = None, + tournament: int | None = None, + last_n_rounds: int | None = None, + distinct_on_round: bool | None = None, + ) -> List[Dict]: + """Fetch submission score history for a model. + + Args: + model_id (str): target model UUID + display_name (str, optional): score metric name filter + version (str, optional): score version filter + day (int, optional): day filter + resolved (bool, optional): resolved-state filter + tournament (int, optional): tournament filter, defaults to the + API instance tournament + last_n_rounds (int, optional): limit by most recent rounds + distinct_on_round (bool, optional): keep only the latest score per + round after applying other filters + + Returns: + list of dicts: list of submission score entries + """ + + query = """ + query($modelId: ID! + $displayName: String + $version: String + $day: Int + $resolved: Boolean + $tournament: Int + $lastNRounds: Int + $distinctOnRound: Boolean) { + submissionScores(modelId: $modelId + displayName: $displayName + version: $version + day: $day + resolved: $resolved + tournament: $tournament + lastNRounds: $lastNRounds + distinctOnRound: $distinctOnRound) { + roundId + submissionId + roundNumber + roundResolveTime + roundScoreTime + roundCloseStakingTime + value + percentile + displayName + version + date + day + resolveDate + resolved + } + } + """ + arguments = { + "modelId": model_id, + "displayName": display_name, + "version": version, + "day": day, + "resolved": resolved, + "tournament": self.tournament_id if tournament is None else tournament, + "lastNRounds": last_n_rounds, + "distinctOnRound": distinct_on_round, + } + scores = self.raw_query(query, arguments)["data"]["submissionScores"] + for score in scores: + utils.replace(score, "roundResolveTime", utils.parse_datetime_string) + utils.replace(score, "roundScoreTime", utils.parse_datetime_string) + utils.replace(score, "roundCloseStakingTime", utils.parse_datetime_string) + utils.replace(score, "date", utils.parse_datetime_string) + utils.replace(score, "resolveDate", utils.parse_datetime_string) + return scores + + def pending_model_payouts(self, tournament: int | None = None) -> Dict: + """Fetch actual and pending payouts for the authenticated user's models. + + Args: + tournament (int, optional): tournament filter, defaults to the API + instance tournament + + Returns: + dict: payout groups with `actual` and `pending` lists + """ + + query = """ + query($tournament: Int!) { + pendingModelPayouts(tournament: $tournament) { + actual { + roundId + roundNumber + roundResolveTime + modelId + modelName + modelDisplayName + payoutNmr + payoutValue + currencySymbol + } + pending { + roundId + roundNumber + roundResolveTime + modelId + modelName + modelDisplayName + payoutNmr + payoutValue + currencySymbol + } + } + } + """ + arguments = { + "tournament": self.tournament_id if tournament is None else tournament + } + payouts = self.raw_query(query, arguments, authorization=True)["data"][ + "pendingModelPayouts" + ] + for payout_type in ["actual", "pending"]: + for payout in payouts[payout_type]: + utils.replace(payout, "roundResolveTime", utils.parse_datetime_string) + utils.replace(payout, "payoutNmr", utils.parse_float_string) + utils.replace(payout, "payoutValue", utils.parse_float_string) + return payouts + def round_model_performances_v2(self, model_id: str): """Fetch round model performance of a user. + DEPRECATED - please use `submission_scores` instead when possible. + Args: model_id (str) @@ -871,6 +1008,13 @@ def round_model_performances_v2(self, model_id: str): * percentile (`float`) * value (`float`): value of the metric """ + warnings.warn( + "`round_model_performances_v2` is deprecated because it relies on " + "`v2RoundModelPerformances`. Use `submission_scores` instead when " + "possible.", + DeprecationWarning, + stacklevel=2, + ) query = """ query($modelId: String! diff --git a/tests/conftest.py b/tests/conftest.py index 134febc..127a61b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,8 +5,71 @@ import sys from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] root_str = str(ROOT) if root_str not in sys.path: sys.path.insert(0, root_str) +from numerapi import base_api + +DEFAULT_FAKE_GRAPHQL_API_URL = "https://fake-api-tournament.numer.ai" + + +def pytest_addoption(parser): + group = parser.getgroup("graphql") + group.addoption( + "--mode", + action="store", + choices=("mock", "integration"), + default="mock", + help=( + "Select whether tests use a fake GraphQL API base URL or a real one. " + "Live API tests only run in integration mode." + ), + ) + group.addoption( + "--api-url", + action="store", + default=None, + help=( + "Base GraphQL API URL to use in tests when --mode=integration. " + "Defaults to numerapi.base_api.API_TOURNAMENT_URL." + ), + ) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", + "live_api: test requires a real GraphQL API backend", + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--mode") == "integration": + return + + skip_live_api = pytest.mark.skip( + reason="requires --mode=integration to run against a real GraphQL API backend" + ) + for item in items: + if "live_api" in item.keywords: + item.add_marker(skip_live_api) + + +@pytest.fixture(scope="session", autouse=True) +def configure_graphql_api_url(pytestconfig): + original_url = base_api.API_TOURNAMENT_URL + mode = pytestconfig.getoption("--mode") + configured_url = pytestconfig.getoption("--api-url") + + if mode == "integration": + base_api.API_TOURNAMENT_URL = configured_url or original_url + else: + base_api.API_TOURNAMENT_URL = DEFAULT_FAKE_GRAPHQL_API_URL + + yield base_api.API_TOURNAMENT_URL + + base_api.API_TOURNAMENT_URL = original_url diff --git a/tests/test_base_api.py b/tests/test_base_api.py index 4d622ec..dd75af1 100644 --- a/tests/test_base_api.py +++ b/tests/test_base_api.py @@ -1,3 +1,6 @@ +import datetime +import decimal +import json import os import pytest import responses @@ -34,8 +37,14 @@ def test__login(api): assert api.token == ("id", "key") +@responses.activate def test_raw_query(api): query = "query {latestNmrPrice {priceUsd}}" + responses.add( + responses.POST, + base_api.API_TOURNAMENT_URL, + json={"data": {"latestNmrPrice": {"priceUsd": "42.00"}}}, + ) result = api.raw_query(query) assert isinstance(result, dict) assert "data" in result @@ -116,3 +125,148 @@ def test_set_submission_webhook(api): 'https://triggerurl' ) assert res + + +@responses.activate +def test_submission_scores(api): + api.tournament_id = 11 + data = { + "data": { + "submissionScores": [ + { + "roundId": "round-1", + "submissionId": "submission-1", + "roundNumber": 123, + "roundResolveTime": "2026-04-01T00:00:00Z", + "roundScoreTime": "2026-03-29T00:00:00Z", + "roundCloseStakingTime": "2026-03-28T00:00:00Z", + "value": 0.12, + "percentile": 0.95, + "displayName": "CORR20", + "version": "v5", + "date": "2026-03-30T00:00:00Z", + "day": 2, + "resolveDate": "2026-04-01T00:00:00Z", + "resolved": True, + } + ] + } + } + responses.add(responses.POST, base_api.API_TOURNAMENT_URL, json=data) + + res = api.submission_scores( + "model-1", + display_name="CORR20", + version="v5", + day=2, + resolved=True, + last_n_rounds=5, + distinct_on_round=True, + ) + + assert len(res) == 1 + assert res[0]["displayName"] == "CORR20" + assert isinstance(res[0]["roundResolveTime"], datetime.datetime) + assert isinstance(res[0]["roundScoreTime"], datetime.datetime) + assert isinstance(res[0]["roundCloseStakingTime"], datetime.datetime) + assert isinstance(res[0]["date"], datetime.datetime) + assert isinstance(res[0]["resolveDate"], datetime.datetime) + + request_body = json.loads(responses.calls[0].request.body) + assert request_body["variables"]["tournament"] == 11 + assert request_body["variables"]["lastNRounds"] == 5 + assert request_body["variables"]["distinctOnRound"] is True + + +@responses.activate +def test_pending_model_payouts(api): + api.token = ("", "") + api.tournament_id = 12 + data = { + "data": { + "pendingModelPayouts": { + "actual": [ + { + "roundId": "round-a", + "roundNumber": 12, + "roundResolveTime": "2026-04-02T00:00:00Z", + "modelId": "model-a", + "modelName": "alpha", + "modelDisplayName": "Alpha", + "payoutNmr": "2.5000", + "payoutValue": "31.20", + "currencySymbol": "$", + } + ], + "pending": [ + { + "roundId": "round-b", + "roundNumber": 13, + "roundResolveTime": "2026-04-09T00:00:00Z", + "modelId": "model-b", + "modelName": "beta", + "modelDisplayName": "Beta", + "payoutNmr": "1.1000", + "payoutValue": "13.73", + "currencySymbol": "$", + } + ], + } + } + } + responses.add(responses.POST, base_api.API_TOURNAMENT_URL, json=data) + + res = api.pending_model_payouts() + + assert len(res["actual"]) == 1 + assert len(res["pending"]) == 1 + assert res["actual"][0]["payoutNmr"] == decimal.Decimal("2.5000") + assert res["pending"][0]["payoutValue"] == decimal.Decimal("13.73") + assert isinstance(res["actual"][0]["roundResolveTime"], datetime.datetime) + assert isinstance(res["pending"][0]["roundResolveTime"], datetime.datetime) + + request_body = json.loads(responses.calls[0].request.body) + assert request_body["variables"]["tournament"] == 12 + + +@responses.activate +def test_round_model_performances_v2_warns(api): + api.tournament_id = 8 + data = { + "data": { + "v2RoundModelPerformances": [ + { + "atRisk": "10.5", + "corrMultiplier": 1.0, + "mmcMultiplier": 0.5, + "roundPayoutFactor": "0.8", + "roundNumber": 456, + "roundOpenTime": "2026-03-01T00:00:00Z", + "roundResolveTime": "2026-03-29T00:00:00Z", + "roundResolved": True, + "roundTarget": "main", + "submissionScores": [ + { + "date": "2026-03-28T00:00:00Z", + "day": 20, + "displayName": "CORR20", + "payoutPending": "0.8", + "payoutSettled": "0.7", + "percentile": 0.9, + "value": 0.12, + } + ], + } + ] + } + } + responses.add(responses.POST, base_api.API_TOURNAMENT_URL, json=data) + + with pytest.warns(DeprecationWarning, match="round_model_performances_v2"): + res = api.round_model_performances_v2("model-1") + + assert len(res) == 1 + assert res[0]["atRisk"] == decimal.Decimal("10.5") + assert isinstance(res[0]["roundOpenTime"], datetime.datetime) + assert isinstance(res[0]["roundResolveTime"], datetime.datetime) + assert res[0]["submissionScores"][0]["payoutPending"] == decimal.Decimal("0.8") diff --git a/tests/test_numerapi.py b/tests/test_numerapi.py index 0b6a80d..fe72a41 100644 --- a/tests/test_numerapi.py +++ b/tests/test_numerapi.py @@ -15,17 +15,20 @@ def api_fixture(): return api +@pytest.mark.live_api def test_get_competitions(api): res = api.get_competitions(tournament=1) assert isinstance(res, list) assert len(res) > 80 +@pytest.mark.live_api def test_get_current_round(api): current_round = api.get_current_round() assert current_round >= 82 +@pytest.mark.live_api @pytest.mark.parametrize("fun", ["get_account", "wallet_transactions"]) def test_unauthorized_requests(api, fun): with pytest.raises(ValueError) as err: @@ -37,6 +40,7 @@ def test_unauthorized_requests(api, fun): "Your session is invalid or has expired." in str(err.value) +@pytest.mark.live_api def test_error_handling(api): # String instead of Int with pytest.raises(ValueError): diff --git a/tests/test_signalsapi.py b/tests/test_signalsapi.py index 9dddafd..0ccaf0f 100644 --- a/tests/test_signalsapi.py +++ b/tests/test_signalsapi.py @@ -13,6 +13,7 @@ def api_fixture(): return api +@pytest.mark.live_api def test_get_leaderboard(api): lb = api.get_leaderboard(1) assert len(lb) == 1