diff --git a/README.md b/README.md index 87729114..6d0447cf 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Here are the tools most likely to be useful to you: - ๐ŸŽฏ **Forecasting Bot:** General forecaster that integrates with the Metaculus AI benchmarking competition and provides a number of utilities. You can forecast with a pre-existing bot or override the class to customize your own (without redoing all the aggregation/API code, etc) - ๐Ÿ”Œ **Metaculus API Wrapper:** for interacting with questions and tournaments - ๐Ÿค– **In-House Metaculus Bots**: You can see all the bots that Metaculus is running on their site in `run_bots.py` +- ๐Ÿ” **Bot Review:** `bot-review` scores your bot's resolved forecasts and shows what it was thinking on the ones it got wrong. See [Reviewing a bot's forecasts](#reviewing-a-bots-forecasts) Here are some other features of the project (not all are documented yet): - **General LLM Wrapper:** A unified interface around litellm with retry logic, the Metaculus proxy, structured outputs, and cost tracking @@ -511,6 +512,49 @@ with MonetaryCostManager(max_cost) as cost_manager: print(f"Current cost: ${current_cost:.2f}") ``` +# Reviewing a bot's forecasts + +`bot-review` builds a table of how your bot's forecasts turned out and attaches the reports it +posted, so you can see what it was thinking on the questions it got wrong. Everything is +read-only: it makes no forecasts, spends nothing on LLMs, and publishes nothing. All it needs +is `METACULUS_TOKEN`. + +```bash +bot-review review --tournament --output review.json --summary review.md +bot-review review --resolved-since 30 # anything that resolved recently +bot-review review --post 44328 44326 # specific questions +bot-review review --from-json review.json --top 3 # re-render without refetching +``` + +The markdown summary gives your leaderboard standing, how many questions were forecast and +scored, and the best and worst questions ranked on whichever score the leaderboard uses โ€” spot +peer for the AI benchmark tournaments, time-averaged peer for the Metaculus Cup. + +The JSON adds per-question detail: the official scores, the bot's forecast, and one `RunTrace` +per run taken from the comment that run posted โ€” each forecaster's prediction, the run time, +cost and minutes where the bot leaves that metadata in. `QuestionOutcome.trace` picks the run +that was standing when the question was spot scored, which is the one that earned the score. + +Reasoning text is not stored. Pull it a piece at a time: + +```bash +bot-review show 44328 --section research +bot-review show 44328 --forecaster R1:F3 --comment 921582 # a specific run +``` + +Or from Python: + +```python +from forecasting_tools.bot_review.outcomes import get_tournament_outcomes +from forecasting_tools.bot_review.summary import build_summary +from forecasting_tools.bot_review.traces import attach_traces, get_trace + +table = get_tournament_outcomes("minibench-2026-06-29") +attach_traces(table) +print(build_summary(table)) +print(get_trace(44328, section="research")) +``` + # Local Development ## Environment Variables diff --git a/code_tests/unit_tests/test_bot_review/test_client_calls.py b/code_tests/unit_tests/test_bot_review/test_client_calls.py new file mode 100644 index 00000000..e92bf720 --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_client_calls.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from unittest.mock import patch + +from forecasting_tools.data_models.comment import Comment +from forecasting_tools.data_models.leaderboard import Leaderboard +from forecasting_tools.helpers.metaculus_client import MetaculusClient + +# Field names and nesting copied from live API responses on 2026-07-24. +COMMENT_JSON = { + "id": 977364, + "on_post": 44676, + "author": {"id": 305884, "username": "a-bot", "is_bot": True}, + "created_at": "2026-07-24T20:49:49.595398Z", + "text": "# SUMMARY\n*Final Prediction*: 20.0%", + "is_private": True, + "included_forecast": {"start_time": "2026-07-24T20:49:44.998152Z"}, + "parent_id": None, +} + +ENTRY_JSON = { + "user": {"id": 303267, "username": "a-bot", "is_bot": True}, + "rank": 17, + "score": 348.50171781642365, + "coverage": 36, + "contribution_count": 36, + "medal": None, + "prize": 0, + "take": 121453.44732099818, + "excluded": False, +} + +LEADERBOARD_JSON = { + "id": 932, + "is_primary_leaderboard": True, + "project_id": 33064, + "project_name": "MiniBench - 2026-06-29", + "project_slug": "minibench-2026-06-29", + "project_type": "question_series", + "score_type": "spot_peer_tournament", + "finalized": True, + "entries": [ENTRY_JSON], + "userEntry": ENTRY_JSON, +} + + +class TestCommentParsing: + def test_reads_the_live_field_names(self): + comment = Comment.from_metaculus_api_json(COMMENT_JSON) + assert comment.id == 977364 + assert comment.on_post == 44676 + assert comment.author_id == 305884 + assert comment.author_username == "a-bot" + assert comment.is_private + assert comment.text.startswith("# SUMMARY") + assert comment.created_at.year == 2026 + + +class TestLeaderboardParsing: + def test_entry_fields(self): + leaderboard = Leaderboard.from_metaculus_api_json(LEADERBOARD_JSON) + assert leaderboard.project_id == 33064 + assert leaderboard.score_type == "spot_peer_tournament" + assert leaderboard.finalized + assert len(leaderboard.entries) == 1 + assert leaderboard.user_entry is not None + assert leaderboard.user_entry.rank == 17 + assert leaderboard.user_entry.user_id == 303267 + assert not leaderboard.user_entry.excluded + + def test_no_entry_when_you_never_forecast_the_tournament(self): + without_entry = {**LEADERBOARD_JSON, "userEntry": None} + assert Leaderboard.from_metaculus_api_json(without_entry).user_entry is None + + def test_community_aggregates_rank_alongside_forecasters(self): + aggregate = { + **ENTRY_JSON, + "user": None, + "aggregation_method": "recency_weighted", + "rank": 16, + } + leaderboard = Leaderboard.from_metaculus_api_json( + {**LEADERBOARD_JSON, "entries": [ENTRY_JSON, aggregate]} + ) + crowd = leaderboard.entries[1] + assert crowd.user_id is None + assert crowd.username is None + assert crowd.aggregation_method == "recency_weighted" + assert leaderboard.entries[0].aggregation_method is None + + def test_a_medalled_entry_keeps_its_medal(self): + medalled = { + **LEADERBOARD_JSON, + "userEntry": {**ENTRY_JSON, "medal": "gold", "rank": 1}, + } + entry = Leaderboard.from_metaculus_api_json(medalled).user_entry + assert entry is not None and entry.medal == "gold" + + +class TestGetOwnComments: + def client(self) -> MetaculusClient: + return MetaculusClient(token="a-token", sleep_seconds_between_requests=0) + + def test_asks_for_private_comments_by_default(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[] + ) as page: + self.client().get_own_comments(user_id=305884) + assert page.call_args.args[0]["is_private"] == "true" + assert page.call_args.args[0]["author"] == 305884 + + def test_post_filter_is_only_sent_when_given(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[] + ) as page: + self.client().get_own_comments(user_id=1) + assert "post" not in page.call_args.args[0] + self.client().get_own_comments(user_id=1, post_id=44676) + assert page.call_args.args[0]["post"] == 44676 + + def test_supplied_user_id_avoids_the_lookup_request(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[] + ), patch.object(MetaculusClient, "get_current_user_id") as lookup: + self.client().get_own_comments(user_id=305884) + lookup.assert_not_called() + + def test_pages_until_a_short_page_arrives(self): + full_page = [ + COMMENT_JSON + ] * MetaculusClient.MAX_COMMENTS_FROM_COMMENT_API_PER_REQUEST + with patch.object( + MetaculusClient, + "_get_comment_page", + side_effect=[full_page, [COMMENT_JSON]], + ) as page: + comments = self.client().get_own_comments(user_id=1) + assert len(comments) == 101 + assert page.call_count == 2 + assert page.call_args_list[1].args[0]["offset"] == 100 + + def test_stops_at_max_comments_without_overfetching(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[COMMENT_JSON] * 5 + ) as page: + comments = self.client().get_own_comments(user_id=1, max_comments=5) + assert len(comments) == 5 + assert page.call_args.args[0]["limit"] == 5 + assert page.call_count == 1 diff --git a/code_tests/unit_tests/test_bot_review/test_outcomes.py b/code_tests/unit_tests/test_bot_review/test_outcomes.py new file mode 100644 index 00000000..464f88f4 --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_outcomes.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import math +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock, patch + +from forecasting_tools.bot_review import outcomes as outcomes_module +from forecasting_tools.bot_review.outcomes import ( + cdf_median, + get_recently_resolved_outcomes, + outcomes_from_post, + scale_internal, + summarize_forecast, +) + + +def make_binary_post(**question_overrides) -> dict: + question = { + "id": 101, + "type": "binary", + "title": "Will X happen?", + "status": "resolved", + "resolution": "yes", + "actual_resolve_time": "2026-07-01T00:00:00Z", + "spot_scoring_time": "2026-06-15T00:00:00Z", + "my_forecasts": { + "latest": { + "forecast_values": [0.3, 0.7], + "start_time": "2026-06-10T00:00:00Z", + }, + "score_data": { + "peer_score": 5.0, + "baseline_score": 10.0, + "coverage": 0.9, + "spot_peer_score": -2.5, + }, + }, + } + question.update(question_overrides) + return { + "id": 1, + "title": "Will X happen?", + "question": question, + "projects": { + "default_project": {"id": 33064, "slug": "a-tournament", "name": "A"} + }, + } + + +class TestScaling: + def test_linear(self): + scaling = {"range_min": 0, "range_max": 100, "zero_point": None} + assert scale_internal(0.5, scaling) == 50 + + def test_log(self): + # range 1..100 with zero_point 0 -> ratio 100, so the midpoint is 10 + scaling = {"range_min": 1, "range_max": 100, "zero_point": 0} + assert math.isclose(scale_internal(0.5, scaling), 10.0) + + def test_missing_range(self): + assert scale_internal(0.5, {}) is None + + def test_cdf_median_interpolates(self): + assert math.isclose(cdf_median([0.0, 0.25, 0.75, 1.0]), 0.5) + + def test_cdf_median_endpoints(self): + assert cdf_median([0.6, 0.8, 1.0]) == 0.0 + assert cdf_median([0.0, 0.1, 0.2]) == 1.0 + + +class TestSummarizeForecast: + def test_binary(self): + assert summarize_forecast("binary", [0.3, 0.7], None, {}) == { + "probability_yes": 0.7 + } + + def test_multiple_choice_zips_options(self): + summary = summarize_forecast("multiple_choice", [0.2, 0.8], ["A", "B"], {}) + assert summary == {"probabilities": {"A": 0.2, "B": 0.8}} + + def test_numeric_median_is_scaled(self): + scaling = {"range_min": 0, "range_max": 100, "zero_point": None} + summary = summarize_forecast("numeric", [0.0, 0.25, 0.75, 1.0], None, scaling) + assert summary is not None + assert math.isclose(summary["median"], 50.0) + + def test_date_gets_an_iso_median(self): + scaling = {"range_min": 0, "range_max": 86400, "zero_point": None} + summary = summarize_forecast("date", [0.0, 0.5, 1.0], None, scaling) + assert summary is not None + assert summary["median_iso"].startswith("1970-01-01T12:00") + + def test_no_forecast(self): + assert summarize_forecast("binary", None, None, {}) is None + + +class TestOutcomesFromPost: + def test_binary_post(self): + (outcome,) = outcomes_from_post(make_binary_post()) + assert outcome.post_id == 1 + assert outcome.question_id == 101 + assert outcome.forecasted + assert outcome.forecast == {"probability_yes": 0.7} + assert outcome.resolution == "yes" + assert outcome.scores is not None + assert outcome.scores.peer_score == 5.0 + assert outcome.scores.spot_peer_score == -2.5 + assert outcome.project_slug == "a-tournament" + assert outcome.url.endswith("/questions/1/") + + def test_resolved_question_with_null_resolution(self): + post = make_binary_post( + type="multiple_choice", + resolution=None, + options=["0", "1"], + my_forecasts={"latest": {"forecast_values": [0.4, 0.6]}, "score_data": {}}, + ) + (outcome,) = outcomes_from_post(post) + assert outcome.status == "resolved" + assert outcome.resolution is None + assert outcome.forecast == {"probabilities": {"0": 0.4, "1": 0.6}} + assert outcome.scores is None + + def test_annulled_question(self): + post = make_binary_post(resolution="annulled", my_forecasts={}) + (outcome,) = outcomes_from_post(post) + assert outcome.was_annulled + assert not outcome.forecasted + + def test_unforecasted_question(self): + (outcome,) = outcomes_from_post(make_binary_post(my_forecasts={})) + assert not outcome.forecasted + assert outcome.forecast is None + assert outcome.scores is None + + def test_group_post_expands_to_one_row_per_subquestion(self): + subquestion = make_binary_post()["question"] + post = { + "id": 2, + "title": "Group", + "projects": {}, + "group_of_questions": { + "questions": [ + dict(subquestion, id=201, label="a"), + dict(subquestion, id=202, label="b"), + ] + }, + } + outcomes = outcomes_from_post(post) + assert [o.question_id for o in outcomes] == [201, 202] + assert [o.group_label for o in outcomes] == ["a", "b"] + assert all(o.post_id == 2 for o in outcomes) + + def test_conditional_post_expands_to_both_branches(self): + child = make_binary_post()["question"] + post = { + "id": 3, + "projects": {}, + "conditional": { + "question_yes": dict(child, id=301), + "question_no": dict(child, id=302), + }, + } + outcomes = outcomes_from_post(post) + assert [o.conditional_branch for o in outcomes] == ["yes", "no"] + + def test_notebooks_are_skipped(self): + assert outcomes_from_post({"id": 4, "notebook": {}}) == [] + + +class TestRecentlyResolved: + def outcome_resolved(self, resolve_time: datetime | None): + (outcome,) = outcomes_from_post(make_binary_post()) + return outcome.model_copy(update={"actual_resolve_time": resolve_time}) + + def test_keeps_only_questions_resolved_inside_the_window(self): + now = datetime.now(tz=timezone.utc) + recent = self.outcome_resolved(now - timedelta(days=2)) + old = self.outcome_resolved(now - timedelta(days=40)) + never = self.outcome_resolved(None) + + client = MagicMock() + client.get_questions_matching_filter = AsyncMock(return_value=[]) + with patch.object( + outcomes_module, + "get_outcomes_for_posts", + return_value=[recent, old, never], + ): + kept = get_recently_resolved_outcomes(7, client) + + assert kept == [recent] + + def test_asks_the_api_for_resolved_questions_it_forecast(self): + client = MagicMock() + client.get_questions_matching_filter = AsyncMock(return_value=[]) + with patch.object(outcomes_module, "get_outcomes_for_posts", return_value=[]): + get_recently_resolved_outcomes(7, client) + + api_filter = client.get_questions_matching_filter.call_args.args[0] + assert api_filter.allowed_statuses == ["resolved"] + assert api_filter.is_previously_forecasted_by_user + assert api_filter.scheduled_resolve_time_gt is not None + + def test_the_api_window_reaches_further_back_than_the_cutoff(self): + # a question scheduled before the cutoff can still resolve inside it + client = MagicMock() + client.get_questions_matching_filter = AsyncMock(return_value=[]) + with patch.object(outcomes_module, "get_outcomes_for_posts", return_value=[]): + get_recently_resolved_outcomes(7, client) + + api_filter = client.get_questions_matching_filter.call_args.args[0] + cutoff = datetime.now(tz=timezone.utc) - timedelta(days=7) + assert api_filter.scheduled_resolve_time_gt < cutoff diff --git a/code_tests/unit_tests/test_bot_review/test_report_parsing.py b/code_tests/unit_tests/test_bot_review/test_report_parsing.py new file mode 100644 index 00000000..fa07568f --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_report_parsing.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +from forecasting_tools.bot_review.report_parsing import ( + final_prediction, + forecaster_rationales, + parse_forecasters, + split_sections, + was_truncated, +) + +# Stock forecasting-tools output: unannotated bullets, two research reports. +TEMPLATE_EXPLANATION = """ +# SUMMARY +*Question*: Will X happen? +*Final Prediction*: 35.0% +*Total Cost*: $0.35 (estimated) + +## Report 1 Summary +### Forecasts +*Forecaster 1*: 30.0% +*Forecaster 2*: 40.0% + +### Research Summary +A short summary of report 1. + +## Report 2 Summary +### Forecasts +*Forecaster 1*: 50.0% +*Forecaster 2*: 60.0% + +### Research Summary +A short summary of report 2. + +# RESEARCH +## Report 1 Research +### Research heading 1 +Body one. +## Report 2 Research +### Research heading 2 +Body two. + +# FORECASTS + +## R1: Forecaster 1 Reasoning +### A heading demoted by the framework +R1F1 reasoning. + +## R1: Forecaster 2 Reasoning +R1F2 reasoning. + +## R2: Forecaster 1 Reasoning +R2F1 reasoning. + +## R2: Forecaster 2 Reasoning +R2F2 reasoning. +""".strip() + +# Some bots add a parenthesised suffix to the bullets; it must not break parsing. +ANNOTATED_EXPLANATION = """ +# SUMMARY +*Question*: Will X happen? +*Final Prediction*: 3.0% + +## Report 1 Summary +### Forecasts +*Forecaster 1 (a-model)*: 3.0% +*Forecaster 2 (another-model)*: 4.0% + +### Research Summary +_Full research in the RESEARCH section below._ + +# RESEARCH +## Report 1 Research +Some news. + +# FORECASTS +## R1: Forecaster 1 Reasoning +First rationale. + +## R1: Forecaster 2 Reasoning +Second rationale. +""".strip() + +MULTILINE_SUMMARY = """ +*Final Prediction*: +- A: 60.00% +- B: 40.00% + +*Total Cost*: disabled + +## Report 1 Summary +### Forecasts +*Forecaster 1*: - A: 62.0% +- B: 38.0% + +*Forecaster 2*: - A: 58.0% +- B: 42.0% + +### Research Summary +_elsewhere_ +""".strip() + + +class TestSplitSections: + def test_finds_the_three_sections(self): + assert set(split_sections(TEMPLATE_EXPLANATION)) == { + "summary", + "research", + "forecasts", + } + + def test_research_stops_where_forecasts_start(self): + research = split_sections(TEMPLATE_EXPLANATION)["research"] + assert "Body two." in research + assert "Forecaster 1 Reasoning" not in research + + def test_missing_sections_are_omitted(self): + # the heading line stays in, as it does on ForecastReport.summary + assert split_sections("# SUMMARY\nonly this") == { + "summary": "# SUMMARY\nonly this" + } + + +class TestParseForecasters: + def test_unannotated_bullets_across_two_reports(self): + summary = split_sections(TEMPLATE_EXPLANATION)["summary"] + assert parse_forecasters(summary) == [ + {"key": "R1:F1", "prediction": "30.0%"}, + {"key": "R1:F2", "prediction": "40.0%"}, + {"key": "R2:F1", "prediction": "50.0%"}, + {"key": "R2:F2", "prediction": "60.0%"}, + ] + + def test_annotated_bullets_still_parse(self): + summary = split_sections(ANNOTATED_EXPLANATION)["summary"] + assert parse_forecasters(summary) == [ + {"key": "R1:F1", "prediction": "3.0%"}, + {"key": "R1:F2", "prediction": "4.0%"}, + ] + + def test_research_summary_is_not_read_as_a_prediction(self): + summary = split_sections(ANNOTATED_EXPLANATION)["summary"] + assert "Full research" not in parse_forecasters(summary)[-1]["prediction"] + + def test_multiline_predictions_are_kept_whole(self): + parsed = parse_forecasters(MULTILINE_SUMMARY) + assert parsed[0]["prediction"] == "- A: 62.0%\n- B: 38.0%" + assert parsed[1]["prediction"] == "- A: 58.0%\n- B: 42.0%" + + +class TestForecasterRationales: + def test_second_research_report_does_not_overwrite_the_first(self): + rationales = forecaster_rationales(TEMPLATE_EXPLANATION) + assert sorted(rationales) == ["R1:F1", "R1:F2", "R2:F1", "R2:F2"] + assert "R1F1 reasoning." in rationales["R1:F1"] + assert "R2F1 reasoning." in rationales["R2:F1"] + + def test_subheadings_stay_with_their_forecaster(self): + rationales = forecaster_rationales(TEMPLATE_EXPLANATION) + assert "A heading demoted by the framework" in rationales["R1:F1"] + assert "R1F2 reasoning" not in rationales["R1:F1"] + + def test_no_forecasts_section(self): + assert forecaster_rationales("# SUMMARY\nnothing else") == {} + + +class TestFinalPrediction: + def test_single_line(self): + summary = split_sections(TEMPLATE_EXPLANATION)["summary"] + assert final_prediction(summary) == "35.0%" + + def test_multiline(self): + assert final_prediction(MULTILINE_SUMMARY) == "- A: 60.00%\n- B: 40.00%" + + def test_absent(self): + assert final_prediction("# SUMMARY\nnothing here") == "" + + +class TestWasTruncated: + def test_detects_the_frameworks_truncation_notice(self): + assert was_truncated( + "# SUMMARY\n---\n The comment size exceeded max size and has been truncated" + ) + assert not was_truncated(TEMPLATE_EXPLANATION) diff --git a/code_tests/unit_tests/test_bot_review/test_summary.py b/code_tests/unit_tests/test_bot_review/test_summary.py new file mode 100644 index 00000000..964b4830 --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_summary.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from forecasting_tools.bot_review.outcomes import ( + OutcomeTable, + QuestionOutcome, + QuestionScores, +) +from forecasting_tools.bot_review.summary import build_summary +from forecasting_tools.data_models.leaderboard import Leaderboard, LeaderboardEntry + + +def make_question( + post_id: int, + spot_peer: float | None = 0.0, + baseline: float | None = 0.0, + peer: float | None = None, + scored: bool = True, + forecasted: bool = True, + resolution: str | None = "yes", + project: str = "CupA", + status: str = "resolved", +) -> QuestionOutcome: + return QuestionOutcome( + post_id=post_id, + question_id=post_id + 100, + title=f"Q{post_id}", + url=f"https://www.metaculus.com/questions/{post_id}/", + question_type="binary", + status=status, + resolution=resolution, + actual_resolve_time=None, + spot_scoring_time=None, + options=None, + forecasted=forecasted, + forecast={"probability_yes": 0.5} if forecasted else None, + forecast_time=None, + scores=( + QuestionScores( + spot_peer_score=spot_peer, + baseline_score=baseline, + peer_score=peer, + coverage=0.5, + ) + if scored + else None + ), + project_id=1, + project_slug=project.lower(), + project_name=project, + ) + + +def make_leaderboard( + rank: int = 3, entries: int = 50, score_type: str = "spot_peer_tournament" +) -> Leaderboard: + def entry(user_id: int | None, rank: int | None) -> LeaderboardEntry: + return LeaderboardEntry( + user_id=user_id, + username=f"bot{user_id}", + aggregation_method=None, + rank=rank, + score=30.0, + coverage=4.0, + contribution_count=4, + medal=None, + prize=None, + excluded=False, + ) + + return Leaderboard( + project_id=1, + project_name="TestCup", + project_slug="testcup", + score_type=score_type, + finalized=False, + entries=[entry(i, i) for i in range(1, entries + 1)], + user_entry=entry(7, rank), + ) + + +def make_table(**overrides) -> OutcomeTable: + fields = { + "generated_at": datetime(2026, 7, 23, tzinfo=timezone.utc), + "user_id": 7, + "project_name": "TestCup", + "leaderboard": make_leaderboard(), + "questions": [ + make_question(1, 20.0, 40.0), + make_question(2, -15.0, -30.0), + make_question(3, 25.0, 50.0), + make_question(4, 0.0, 5.0), + make_question(5, scored=False, resolution="annulled"), + make_question(6, scored=False, forecasted=False, resolution=None), + ], + } + fields.update(overrides) + return OutcomeTable(**fields) + + +def test_counts(): + report = build_summary(make_table()) + assert "Questions: 6" in report + assert "Forecasted: 5" in report + assert "Scored: 4" in report + assert "Forecasted but unscored: 1 (1 resolved without a score)" in report + + +def test_single_project_still_shows_breakdown(): + report = build_summary(make_table()) + assert "### By tournament" in report + assert "- CupA: 6 questions (4 scored)" in report + + +def test_multi_project_breakdown_counts_questions_and_scored(): + questions = make_table().questions + for index in (0, 4): + questions[index] = questions[index].model_copy( + update={"project_name": "CupB", "project_slug": "cupb"} + ) + report = build_summary(make_table(questions=questions)) + assert "- CupA: 4 questions (3 scored)" in report + assert "- CupB: 2 questions (1 scored)" in report + + +def test_annulled_listed_with_resolution(): + report = build_summary(make_table()) + assert "Q5" in report and "resolution: annulled" in report + + +def test_best_worst_ordering(): + report = build_summary(make_table(), top_n=2) + best = report.split("## Best")[1].split("## Worst")[0] + worst = report.split("## Worst")[1] + assert best.index("Q3") < best.index("Q1") + assert worst.index("Q2") < worst.index("Q4") + assert "Q2" not in best + assert "Q3" not in worst + + +def test_signed_formatting_and_leaderboard(): + report = build_summary(make_table()) + assert "+25.0 spot peer" in report + assert "-15.0 spot peer" in report + assert "Rank: **3 / 50** (spot_peer_tournament)" in report + + +def test_a_peer_tournament_is_ranked_on_peer_score(): + questions = [ + make_question(1, spot_peer=90.0, peer=-5.0), + make_question(2, spot_peer=-90.0, peer=10.0), + ] + report = build_summary( + make_table( + questions=questions, + leaderboard=make_leaderboard(score_type="peer_tournament"), + ) + ) + assert "## Best 2 (by peer score)" in report + assert report.split("## Best")[1].index("Q2") < report.split("## Best")[1].index( + "Q1" + ) + assert "+10.0 peer" in report + + +def test_an_unknown_score_type_falls_back_to_spot_peer(): + report = build_summary( + make_table(leaderboard=make_leaderboard(score_type="comment_insight")) + ) + assert "## Best 4 (by spot peer score)" in report + + +def test_mean_coverage_over_scored(): + report = build_summary(make_table()) + assert "Mean coverage: 0.50 (over 4 scored questions)" in report + + +def test_open_unscored_question_not_listed_as_resolved(): + questions = make_table().questions + questions[4] = questions[4].model_copy(update={"status": "open"}) + report = build_summary(make_table(questions=questions)) + assert "Forecasted but unscored: 1 (0 resolved without a score)" in report + assert "Q5" not in report + + +def test_no_leaderboard_entry(): + assert "No leaderboard entry found." in build_summary(make_table(leaderboard=None)) + + +def test_questions_without_a_spot_peer_score_are_not_ranked(): + questions = [make_question(1, spot_peer=None, baseline=40.0), make_question(2, 5.0)] + report = build_summary(make_table(questions=questions)) + assert "Scored: 2" in report + assert "## Best 1 (by spot peer score)" in report + assert "Q1" not in report.split("## Best")[1] + + +def test_a_table_saved_to_json_renders_the_same_report(): + table = make_table() + reloaded = OutcomeTable.model_validate_json(table.model_dump_json()) + assert build_summary(reloaded) == build_summary(table) + + +def test_no_scored_questions(): + questions = [make_question(1, scored=False, status="open")] + report = build_summary(make_table(questions=questions, leaderboard=None)) + assert "## Best 0 (by spot peer score)" in report + assert "Mean coverage" not in report diff --git a/code_tests/unit_tests/test_bot_review/test_traces.py b/code_tests/unit_tests/test_bot_review/test_traces.py new file mode 100644 index 00000000..48b7a517 --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_traces.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +from forecasting_tools.bot_review.outcomes import OutcomeTable, QuestionOutcome +from forecasting_tools.bot_review.traces import attach_traces, get_trace, reduce_comment +from forecasting_tools.data_models.comment import Comment + +RUN_TIME = datetime(2026, 6, 30, 10, tzinfo=timezone.utc) + +EXPLANATION = """ +# SUMMARY +*Question*: Q1 +*Final Prediction*: 20.0% +*Total Cost*: $0.0706 (estimated) +*Time Spent*: 0.28 minutes +*Bot Name*: TemplateBot + +## Report 1 Summary +### Forecasts +*Forecaster 1*: 20.0% +*Forecaster 2 (gpt-5)*: 30.0% + +# RESEARCH +## Report 1 Research +the research + +# FORECASTS +## R1: Forecaster 1 Reasoning +the first rationale +## R1: Forecaster 2 Reasoning +the second rationale +""" + + +def make_comment( + comment_id: int = 1, + post_id: int = 1, + created_at: datetime = RUN_TIME, + text: str = EXPLANATION, +) -> Comment: + return Comment( + id=comment_id, + on_post=post_id, + author_id=7, + author_username="bot", + created_at=created_at, + text=text, + is_private=True, + ) + + +def make_question(post_id: int = 1, title: str = "Q1") -> QuestionOutcome: + return QuestionOutcome( + post_id=post_id, + question_id=post_id + 100, + title=title, + url=f"https://www.metaculus.com/questions/{post_id}/", + question_type="binary", + status="resolved", + resolution="yes", + actual_resolve_time=None, + spot_scoring_time=None, + options=None, + forecasted=True, + forecast={"probability_yes": 0.2}, + forecast_time=None, + scores=None, + project_id=1, + project_slug="testcup", + project_name="TestCup", + ) + + +def make_table(questions) -> OutcomeTable: + return OutcomeTable( + generated_at=RUN_TIME, user_id=7, questions=questions, project_name="TestCup" + ) + + +class TestReduceComment: + def test_reads_the_metadata_the_bot_writes_into_its_report(self): + trace = reduce_comment(make_comment()) + assert trace.comment_id == 1 + assert trace.run_time == RUN_TIME + assert trace.question_text == "Q1" + assert trace.cost == 0.0706 + assert trace.minutes == 0.28 + assert not trace.truncated + + def test_reads_each_forecaster_prediction(self): + trace = reduce_comment(make_comment()) + assert [f["key"] for f in trace.forecasters] == ["R1:F1", "R1:F2"] + assert trace.forecasters[1]["prediction"] == "30.0%" + + def test_missing_metadata_is_none_not_an_error(self): + trace = reduce_comment(make_comment(text="# SUMMARY\nnothing useful here")) + assert trace.cost is None + assert trace.minutes is None + assert trace.question_text is None + assert trace.forecasters == [] + + def test_a_truncated_comment_is_flagged(self): + text = "# SUMMARY\n\n---\n\n The comment size exceeded max size and has been truncated" + assert reduce_comment(make_comment(text=text)).truncated + + +class TestAttachTraces: + def client_returning(self, *comments) -> MagicMock: + """A client whose comments are all private, as most bots post them.""" + client = MagicMock() + client.get_own_comments.side_effect = lambda **kwargs: ( + list(comments) if kwargs.get("is_private") else [] + ) + return client + + def test_comments_are_read_in_bulk_not_once_per_post(self): + questions = [make_question(1), make_question(2), make_question(3)] + client = self.client_returning(make_comment()) + attach_traces(make_table(questions), client) + assert client.get_own_comments.call_count == 2 + assert [ + c.kwargs["is_private"] for c in client.get_own_comments.call_args_list + ] == [ + True, + False, + ] + + def test_public_comments_are_read_too(self): + question = make_question(1) + client = MagicMock() + client.get_own_comments.side_effect = lambda **kwargs: ( + [] if kwargs.get("is_private") else [make_comment(comment_id=9)] + ) + attach_traces(make_table([question]), client) + assert [t.comment_id for t in question.traces] == [9] + + def test_comments_on_other_posts_are_ignored(self): + question = make_question(1) + client = self.client_returning(make_comment(post_id=1), make_comment(post_id=2)) + attach_traces(make_table([question]), client) + assert len(question.traces) == 1 + + def test_a_group_post_splits_its_comments_by_question_text(self): + first = make_question(title="Q1") + second = make_question(title="other") + client = self.client_returning( + make_comment(comment_id=1), + make_comment( + comment_id=2, + text=EXPLANATION.replace("*Question*: Q1", "*Question*: other"), + ), + ) + attach_traces(make_table([first, second]), client) + assert [t.comment_id for t in first.traces] == [1] + assert [t.comment_id for t in second.traces] == [2] + + def test_a_single_question_post_keeps_every_run(self): + question = make_question(1) + client = self.client_returning( + make_comment(comment_id=1), make_comment(comment_id=2) + ) + attach_traces(make_table([question]), client) + assert len(question.traces) == 2 + + +class TestSelectingTheScoredRun: + def question_with_runs(self, spot_offset_hours: float | None, *run_offsets: float): + question = make_question(1).model_copy( + update={ + "spot_scoring_time": ( + None + if spot_offset_hours is None + else RUN_TIME + timedelta(hours=spot_offset_hours) + ) + } + ) + question.traces = [ + reduce_comment( + make_comment( + comment_id=i, created_at=RUN_TIME + timedelta(hours=offset) + ) + ) + for i, offset in enumerate(run_offsets) + ] + return question + + def test_the_latest_run_standing_at_spot_time_is_the_one_that_scored(self): + question = self.question_with_runs(5, 0, 2, 9) + assert question.trace is not None + assert question.trace.comment_id == 1 + + def test_a_run_after_spot_time_did_not_earn_the_score(self): + question = self.question_with_runs(5, 9) + assert question.traces + assert question.trace is None + + def test_without_a_spot_time_the_latest_run_is_used(self): + question = self.question_with_runs(None, 0, 9) + assert question.trace is not None + assert question.trace.comment_id == 1 + + def test_no_runs_at_all(self): + assert make_question(1).trace is None + + def test_traces_survive_a_json_round_trip(self): + table = make_table([self.question_with_runs(5, 0)]) + reloaded = OutcomeTable.model_validate_json(table.model_dump_json()) + assert reloaded.questions[0].trace is not None + assert reloaded.questions[0].trace.forecasters[0]["key"] == "R1:F1" + + +class TestGetTrace: + def client_returning(self, *comments) -> MagicMock: + """A client whose comments are all private, as most bots post them.""" + client = MagicMock() + client.get_own_comments.side_effect = lambda **kwargs: ( + list(comments) if kwargs.get("is_private") else [] + ) + return client + + def test_returns_the_asked_for_section(self): + client = self.client_returning(make_comment()) + assert "the research" in get_trace(1, "research", client=client) + + def test_returns_one_forecaster_rationale(self): + client = self.client_returning(make_comment()) + text = get_trace(1, forecaster="R1:F2", client=client) + assert "the second rationale" in text + assert "the first rationale" not in text + + def test_reads_the_newest_comment_on_the_post(self): + client = self.client_returning( + make_comment(comment_id=1, text=EXPLANATION.replace("the research", "old")), + make_comment(comment_id=2, created_at=RUN_TIME + timedelta(hours=1)), + ) + assert "the research" in get_trace(1, "research", client=client) + + def test_reads_the_run_that_scored_when_given_its_comment_id(self): + client = self.client_returning( + make_comment( + comment_id=1, text=EXPLANATION.replace("the research", "the scored run") + ), + make_comment(comment_id=2, created_at=RUN_TIME + timedelta(hours=1)), + ) + assert "the scored run" in get_trace(1, "research", comment_id=1, client=client) + + def test_an_unknown_comment_id_returns_nothing(self): + client = self.client_returning(make_comment(comment_id=1)) + assert get_trace(1, comment_id=99, client=client) == "" + + def test_a_post_with_no_comments(self): + assert get_trace(1, client=self.client_returning()) == "" diff --git a/forecasting_tools/bot_review/__init__.py b/forecasting_tools/bot_review/__init__.py new file mode 100644 index 00000000..640ec129 --- /dev/null +++ b/forecasting_tools/bot_review/__init__.py @@ -0,0 +1 @@ +"""Tools for reviewing how a bot performed on the questions it forecast.""" diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py new file mode 100644 index 00000000..ff853efe --- /dev/null +++ b/forecasting_tools/bot_review/cli.py @@ -0,0 +1,141 @@ +"""Command line entry point for reviewing a bot's forecasts.""" + +from __future__ import annotations + +import argparse +import logging +from datetime import datetime, timezone + +from dotenv import find_dotenv, load_dotenv + +from forecasting_tools.bot_review.outcomes import ( + OutcomeTable, + get_outcomes_for_posts, + get_recently_resolved_outcomes, + get_tournament_outcomes, +) +from forecasting_tools.bot_review.summary import build_summary +from forecasting_tools.bot_review.traces import attach_traces, get_trace +from forecasting_tools.helpers.metaculus_client import MetaculusClient + + +def _write_and_print(table: OutcomeTable, args: argparse.Namespace) -> None: + report = build_summary(table, top_n=args.top) + print(report) + if args.output: + with open(args.output, "w") as file: + file.write(table.model_dump_json(indent=2)) + print(f"wrote {args.output}") + if args.summary: + with open(args.summary, "w") as file: + file.write(report) + print(f"wrote {args.summary}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Review how a bot's forecasts did") + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--seconds-between-requests", + type=float, + default=0.7, + help="delay between Metaculus requests", + ) + commands = parser.add_subparsers(dest="command", required=True) + + review = commands.add_parser( + "review", + parents=[common], + help="score the bot's questions and write the summary", + ) + source = review.add_mutually_exclusive_group(required=True) + source.add_argument("--tournament", help="tournament slug or id") + source.add_argument("--post", type=int, nargs="+", metavar="POST_ID") + source.add_argument( + "--resolved-since", + type=int, + metavar="DAYS", + help="your questions that resolved in the last DAYS days, any tournament", + ) + source.add_argument( + "--from-json", metavar="FILE", help="re-render a table saved with --output" + ) + review.add_argument( + "--include-unforecasted", + action="store_true", + help="also include questions in the tournament that the bot never forecast", + ) + review.add_argument("--output", help="write the full table to this json file") + review.add_argument("--summary", help="write the markdown report to this file") + review.add_argument( + "--top", type=int, default=10, help="how many best and worst questions to list" + ) + + show = commands.add_parser( + "show", parents=[common], help="print part of one report the bot posted" + ) + show.add_argument("post_id", type=int) + show.add_argument( + "--section", + default="research", + choices=["summary", "research", "forecasts"], + help="which section to print", + ) + show.add_argument( + "--forecaster", metavar="KEY", help="print one rationale, e.g. R1:F2" + ) + show.add_argument( + "--comment", + type=int, + metavar="ID", + help="which run to read, from a trace's comment_id (default: the latest)", + ) + + args = parser.parse_args() + logging.basicConfig(level=logging.WARNING) + load_dotenv(find_dotenv(usecwd=True)) + + if args.command == "review" and args.from_json: + with open(args.from_json) as file: + table = OutcomeTable.model_validate_json(file.read()) + print(f"\nreviewing forecasts made by user {table.user_id}") + _write_and_print(table, args) + return + + client = MetaculusClient( + sleep_seconds_between_requests=args.seconds_between_requests + ) + if args.command == "show": + print( + get_trace(args.post_id, args.section, args.forecaster, args.comment, client) + ) + return + + user_id = client.get_current_user_id() + print(f"\nreviewing forecasts made by user {user_id}") + if args.tournament: + table = get_tournament_outcomes( + args.tournament, + client=client, + forecasted_only=not args.include_unforecasted, + ) + else: + if args.post: + outcomes = get_outcomes_for_posts(args.post, client) + project_name = "selected questions" + else: + outcomes = get_recently_resolved_outcomes(args.resolved_since, client) + project_name = f"questions resolved in the last {args.resolved_since} days" + table = OutcomeTable( + generated_at=datetime.now(tz=timezone.utc), + user_id=user_id, + questions=outcomes, + project_name=project_name, + ) + + attach_traces(table, client) + _write_and_print(table, args) + + +if __name__ == "__main__": + main() diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py new file mode 100644 index 00000000..1e1e0391 --- /dev/null +++ b/forecasting_tools/bot_review/outcomes.py @@ -0,0 +1,306 @@ +"""Build a table of how a bot's forecasts turned out. + +One row per question in a tournament: the bot's latest forecast, how the +question resolved, and the scores Metaculus computed for it. +""" + +from __future__ import annotations + +import asyncio +import logging +from datetime import datetime, timedelta, timezone +from typing import Any + +from pydantic import BaseModel + +from forecasting_tools.data_models.leaderboard import Leaderboard +from forecasting_tools.helpers.metaculus_client import ApiFilter, MetaculusClient + +logger = logging.getLogger(__name__) + +CONTINUOUS_TYPES = ("numeric", "date", "discrete") +MAX_QUESTIONS_PER_TOURNAMENT = 1000 +LATE_RESOLUTION_ALLOWANCE = timedelta(days=14) + + +class QuestionScores(BaseModel): + """Official scores from ``my_forecasts.score_data``.""" + + peer_score: float | None = None + baseline_score: float | None = None + spot_peer_score: float | None = None + spot_baseline_score: float | None = None + coverage: float | None = None + weighted_coverage: float | None = None + relative_legacy_score: float | None = None + + +class RunTrace(BaseModel): + """One run of the bot on one question, as recorded in the comment it posted.""" + + comment_id: int + run_time: datetime + question_text: str | None + forecasters: list[dict[str, Any]] + cost: float | None + minutes: float | None + truncated: bool + + +class QuestionOutcome(BaseModel): + """A question, the bot's forecast on it, and the result.""" + + post_id: int + question_id: int + title: str | None + url: str + question_type: str | None + status: str | None + resolution: str | None + actual_resolve_time: datetime | None + spot_scoring_time: datetime | None + options: list[str] | None + group_label: str | None = None + conditional_branch: str | None = None + forecasted: bool + forecast: dict[str, Any] | None + forecast_time: datetime | None + scores: QuestionScores | None + project_id: int | None + project_slug: str | None + project_name: str | None + traces: list[RunTrace] = [] + + @property + def was_annulled(self) -> bool: + return self.resolution == "annulled" + + @property + def trace(self) -> RunTrace | None: + """The run standing when the question was spot scored, if any was.""" + runs = sorted(self.traces, key=lambda run: run.run_time) + if self.spot_scoring_time is None: + return runs[-1] if runs else None + standing = [run for run in runs if run.run_time <= self.spot_scoring_time] + return standing[-1] if standing else None + + +class OutcomeTable(BaseModel): + """A set of questions, the bot's forecasts on them, and its standing.""" + + generated_at: datetime + user_id: int + questions: list[QuestionOutcome] + project_id: int | None = None + project_slug: str | None = None + project_name: str | None = None + leaderboard: Leaderboard | None = None + + +def scale_internal(location: float, scaling: dict) -> float | None: + """Map an internal [0, 1] location onto the question's own units.""" + range_min, range_max = scaling.get("range_min"), scaling.get("range_max") + zero_point = scaling.get("zero_point") + if range_min is None or range_max is None: + return None + if zero_point is not None: + ratio = (range_max - zero_point) / (range_min - zero_point) + return range_min + (range_max - range_min) * (ratio**location - 1) / (ratio - 1) + return range_min + location * (range_max - range_min) + + +def cdf_median(cdf: list[float]) -> float: + """The internal [0, 1] location where the CDF crosses 0.5.""" + for index, value in enumerate(cdf): + if value >= 0.5: + if index == 0: + return 0.0 + below = cdf[index - 1] + fraction = 0.0 if value == below else (0.5 - below) / (value - below) + return (index - 1 + fraction) / (len(cdf) - 1) + return 1.0 + + +def summarize_forecast( + question_type: str | None, + forecast_values: list[float] | None, + options: list[str] | None, + scaling: dict, +) -> dict[str, Any] | None: + """Reduce raw ``forecast_values`` to a readable summary of the forecast.""" + if not forecast_values: + return None + if question_type == "binary": + probability = ( + forecast_values[-1] if len(forecast_values) == 2 else forecast_values[0] + ) + return {"probability_yes": probability} + if question_type == "multiple_choice": + return {"probabilities": dict(zip(options or [], forecast_values))} + if question_type in CONTINUOUS_TYPES: + median = scale_internal(cdf_median(forecast_values), scaling) + summary: dict[str, Any] = {"median": median} + if question_type == "date" and median is not None: + summary["median_iso"] = datetime.fromtimestamp( + median, tz=timezone.utc + ).isoformat() + return summary + return {"values": forecast_values} + + +def _question_outcome( + post_json: dict, question_json: dict, **extra: Any +) -> QuestionOutcome: + post_id = post_json["id"] + question_type = question_json.get("type") + options = question_json.get("options") + scaling = question_json.get("scaling") or {} + project = (post_json.get("projects") or {}).get("default_project") or {} + my_forecasts = question_json.get("my_forecasts") or {} + latest = my_forecasts.get("latest") or {} + score_data = my_forecasts.get("score_data") or None + return QuestionOutcome( + post_id=post_id, + question_id=question_json["id"], + title=question_json.get("title") or post_json.get("title"), + url=f"https://www.metaculus.com/questions/{post_id}/", + question_type=question_type, + status=question_json.get("status") or post_json.get("status"), + resolution=question_json.get("resolution"), + actual_resolve_time=question_json.get("actual_resolve_time"), + spot_scoring_time=question_json.get("spot_scoring_time"), + options=options, + forecasted=bool(latest), + forecast=summarize_forecast( + question_type, latest.get("forecast_values"), options, scaling + ), + forecast_time=latest.get("start_time"), + scores=QuestionScores(**score_data) if score_data else None, + project_id=project.get("id"), + project_slug=project.get("slug"), + project_name=project.get("name"), + **extra, + ) + + +def outcomes_from_post(post_json: dict) -> list[QuestionOutcome]: + """One outcome per question on a post, unpacking groups and conditionals.""" + if post_json.get("notebook"): + return [] + if post_json.get("question"): + return [_question_outcome(post_json, post_json["question"])] + group = post_json.get("group_of_questions") + if group: + return [ + _question_outcome(post_json, question, group_label=question.get("label")) + for question in group.get("questions", []) + ] + conditional = post_json.get("conditional") + if conditional: + return [ + _question_outcome( + post_json, conditional[branch], conditional_branch=branch.split("_")[1] + ) + for branch in ("question_yes", "question_no") + if conditional.get(branch) + ] + return [] + + +def get_outcomes_for_posts( + post_ids: list[int], client: MetaculusClient | None = None +) -> list[QuestionOutcome]: + """ + Fetch outcomes for specific posts, one request each. + + :param post_ids: post ids, as they appear in question urls + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + outcomes: list[QuestionOutcome] = [] + for post_id in post_ids: + fetched = client.get_question_by_post_id( + post_id, group_question_mode="unpack_subquestions" + ) + questions = fetched if isinstance(fetched, list) else [fetched] + outcomes.extend(outcomes_from_post(questions[0].api_json)) + return outcomes + + +def get_recently_resolved_outcomes( + days: int, client: MetaculusClient | None = None +) -> list[QuestionOutcome]: + """ + Fetch the bot's forecasts on questions that resolved in the last few days. + + :param days: how far back to look + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + cutoff = datetime.now(tz=timezone.utc) - timedelta(days=days) + api_filter = ApiFilter( + allowed_statuses=["resolved"], + is_previously_forecasted_by_user=True, + scheduled_resolve_time_gt=cutoff - LATE_RESOLUTION_ALLOWANCE, + group_question_mode="unpack_subquestions", + ) + listed_questions = asyncio.run( + client.get_questions_matching_filter( + api_filter, + num_questions=MAX_QUESTIONS_PER_TOURNAMENT, + error_if_question_target_missed=False, + ) + ) + post_ids = list(dict.fromkeys(question.id_of_post for question in listed_questions)) + logger.info(f"Building outcomes for {len(post_ids)} posts resolved since {cutoff}") + outcomes = get_outcomes_for_posts(post_ids, client) # type: ignore[arg-type] + return [ + outcome + for outcome in outcomes + if outcome.actual_resolve_time is not None + and outcome.actual_resolve_time >= cutoff + ] + + +def get_tournament_outcomes( + tournament: int | str, + client: MetaculusClient | None = None, + forecasted_only: bool = False, +) -> OutcomeTable: + """ + Fetch the questions in a tournament and how the bot did on each. + + :param tournament: tournament slug or id + :param client: client to fetch with, created from the environment if not given + :param forecasted_only: skip questions the bot never forecast + """ + client = client or MetaculusClient() + api_filter = ApiFilter( + allowed_tournaments=[tournament], + group_question_mode="unpack_subquestions", + is_previously_forecasted_by_user=True if forecasted_only else None, + ) + listed_questions = asyncio.run( + client.get_questions_matching_filter( + api_filter, + num_questions=MAX_QUESTIONS_PER_TOURNAMENT, + error_if_question_target_missed=False, + ) + ) + post_ids = list(dict.fromkeys(question.id_of_post for question in listed_questions)) + logger.info(f"Building outcomes for {len(post_ids)} posts in {tournament}") + + outcomes = get_outcomes_for_posts(post_ids, client) # type: ignore[arg-type] + project_id = outcomes[0].project_id if outcomes else None + leaderboard = ( + client.get_project_leaderboard(project_id) if project_id is not None else None + ) + return OutcomeTable( + generated_at=datetime.now(tz=timezone.utc), + project_id=project_id, + project_slug=outcomes[0].project_slug if outcomes else None, + project_name=outcomes[0].project_name if outcomes else None, + user_id=client.get_current_user_id(), + leaderboard=leaderboard, + questions=outcomes, + ) diff --git a/forecasting_tools/bot_review/report_parsing.py b/forecasting_tools/bot_review/report_parsing.py new file mode 100644 index 00000000..76ed6a2e --- /dev/null +++ b/forecasting_tools/bot_review/report_parsing.py @@ -0,0 +1,92 @@ +"""Pull structure back out of a report explanation. + +Reads the format ``ForecastBot._create_comment`` produces: a SUMMARY listing +each forecaster's prediction, the RESEARCH behind them, and one FORECASTS +subsection per forecaster. Works on a saved report or on the comment text +posted to Metaculus, which are the same string. + +Forecasters are keyed ``R:F``. +""" + +from __future__ import annotations + +import re + +from forecasting_tools.data_models.markdown_tree import MarkdownTree + +SECTIONS = ("summary", "research", "forecasts") +REPORT_PATTERN = re.compile(r"^## Report (\d+) Summary\s*$", re.MULTILINE) +FORECASTER_PATTERN = re.compile(r"^\*Forecaster (\d+)(?: \([^)]*\))?\*:", re.MULTILINE) +RATIONALE_TITLE_PATTERN = re.compile(r"^R(\d+): Forecaster (\d+) Reasoning$") +FINAL_PREDICTION_PATTERN = re.compile( + r"^\*Final Prediction\*:(.*?)(?=^\*[A-Z]|^## |\Z)", re.MULTILINE | re.DOTALL +) + + +def split_sections(explanation: str) -> dict[str, str]: + """The SUMMARY / RESEARCH / FORECASTS blocks, by lowercased name.""" + return { + section.title.strip().lower(): section.text_of_section_and_subsections.strip() + for section in MarkdownTree.turn_markdown_into_report_sections(explanation) + if section.title and section.title.strip().lower() in SECTIONS + } + + +def forecaster_rationales(explanation: str) -> dict[str, str]: + """Full reasoning text for each forecaster.""" + rationales = {} + for section in MarkdownTree.turn_markdown_into_report_sections(explanation): + if not section.title or section.title.strip().lower() != "forecasts": + continue + for subsection in section.sub_sections: + match = RATIONALE_TITLE_PATTERN.match((subsection.title or "").strip()) + if match: + key = f"R{match.group(1)}:F{match.group(2)}" + rationales[key] = subsection.text_of_section_and_subsections.strip() + return rationales + + +def _bodies_between(text: str, marks: list[tuple[int, int]]) -> list[str]: + return [ + text[end : (marks[i + 1][0] if i + 1 < len(marks) else len(text))].strip() + for i, (_, end) in enumerate(marks) + ] + + +def forecasts_block(summary: str) -> str: + """The ``### Forecasts`` block of one report's summary.""" + match = re.search(r"^### Forecasts\s*$", summary, re.MULTILINE) + if not match: + return "" + rest = summary[match.end() :] + following = re.search(r"^### ", rest, re.MULTILINE) + return rest[: following.start()] if following else rest + + +def parse_forecasters(summary: str) -> list[dict]: + """Each forecaster's prediction as it was written in the summary.""" + reports = list(REPORT_PATTERN.finditer(summary)) + blocks = _bodies_between(summary, [(m.start(), m.end()) for m in reports]) + forecasters = [] + for report, block in zip(reports, blocks): + bullets = forecasts_block(block) + matches = list(FORECASTER_PATTERN.finditer(bullets)) + bodies = _bodies_between(bullets, [(m.start(), m.end()) for m in matches]) + forecasters += [ + { + "key": f"R{report.group(1)}:F{match.group(1)}", + "prediction": body, + } + for match, body in zip(matches, bodies) + ] + return forecasters + + +def final_prediction(summary: str) -> str: + match = FINAL_PREDICTION_PATTERN.search(summary) + return match.group(1).strip() if match else "" + + +def was_truncated(explanation: str) -> bool: + """Whether the explanation hit the comment size cap and lost its tail.""" + return "has been truncated" in explanation diff --git a/forecasting_tools/bot_review/summary.py b/forecasting_tools/bot_review/summary.py new file mode 100644 index 00000000..4c00424f --- /dev/null +++ b/forecasting_tools/bot_review/summary.py @@ -0,0 +1,150 @@ +"""Turn an outcome table into a markdown report. + +Covers the bot's leaderboard standing, how many questions it forecast and +scored, and the questions it did best and worst on. +""" + +from __future__ import annotations + +from collections import Counter + +from forecasting_tools.bot_review.outcomes import OutcomeTable, QuestionOutcome + +METRIC_BY_SCORE_TYPE = { + "peer_tournament": "peer_score", + "spot_peer_tournament": "spot_peer_score", + "spot_baseline_tournament": "spot_baseline_score", + "relative_legacy_tournament": "relative_legacy_score", +} +DEFAULT_METRIC = "spot_peer_score" + + +def _rank_metric(table: OutcomeTable) -> str: + """The per-question score the table's leaderboard ranks on.""" + if table.leaderboard is None: + return DEFAULT_METRIC + return METRIC_BY_SCORE_TYPE.get(table.leaderboard.score_type, DEFAULT_METRIC) + + +def _metric_label(metric: str) -> str: + return metric.removesuffix("_score").replace("_", " ") + + +def _signed(value: float | None) -> str: + return "n/a" if value is None else f"{value:+.1f}" + + +def _project_label(outcome: QuestionOutcome) -> str: + return outcome.project_name or outcome.project_slug or str(outcome.project_id) + + +def _question_line(rank: int, outcome: QuestionOutcome, metric: str) -> str: + assert outcome.scores is not None + return ( + f"{rank}. {_signed(getattr(outcome.scores, metric))} {_metric_label(metric)} ยท " + f"{_signed(outcome.scores.baseline_score)} baseline ยท " + f"{outcome.question_type} ยท [{outcome.title}]({outcome.url})" + ) + + +def _overall_lines(table: OutcomeTable, scored: list[QuestionOutcome]) -> list[str]: + lines = ["## Overall", ""] + leaderboard = table.leaderboard + entry = leaderboard.user_entry if leaderboard else None + if leaderboard is not None and entry is not None: + lines += [ + ( + f"- Rank: **{entry.rank} / {len(leaderboard.entries)}** " + f"({leaderboard.score_type})" + ), + ( + f"- Score: **{entry.score:.2f}**" + if entry.score is not None + else "- Score: n/a" + ), + f"- Medal: {entry.medal or 'none'}", + ] + else: + lines.append("- No leaderboard entry found.") + coverages = [ + outcome.scores.coverage + for outcome in scored + if outcome.scores is not None and outcome.scores.coverage is not None + ] + if coverages: + mean_coverage = sum(coverages) / len(coverages) + lines.append( + f"- Mean coverage: {mean_coverage:.2f} (over {len(coverages)} scored questions)" + ) + return lines + + +def build_summary(table: OutcomeTable, top_n: int = 10) -> str: + """ + Render an outcome table as a markdown report. + + :param table: the questions and standing to report on + :param top_n: how many questions to list under best and worst + """ + questions = table.questions + forecasted = [outcome for outcome in questions if outcome.forecasted] + scored = [outcome for outcome in forecasted if outcome.scores is not None] + unscored = [outcome for outcome in forecasted if outcome.scores is None] + resolved_unscored = [ + outcome for outcome in unscored if outcome.status == "resolved" + ] + metric = _rank_metric(table) + ranked = sorted( + ( + outcome + for outcome in scored + if outcome.scores is not None + and getattr(outcome.scores, metric) is not None + ), + key=lambda outcome: getattr(outcome.scores, metric), + reverse=True, + ) + + lines = [ + f"# Review: {table.project_name or 'forecast outcomes'}", + "", + f"user {table.user_id} โ€” generated {table.generated_at.isoformat()}", + "", + ] + lines += _overall_lines(table, scored) + lines += [ + "", + "## Participation", + "", + f"- Questions: {len(questions)}", + f"- Forecasted: {len(forecasted)}", + f"- Scored: {len(scored)}", + ( + f"- Forecasted but unscored: {len(unscored)} " + f"({len(resolved_unscored)} resolved without a score)" + ), + ] + for outcome in resolved_unscored: + lines.append( + f" - [{outcome.title}]({outcome.url}) โ€” resolution: {outcome.resolution}" + ) + lines.append( + f"- Traced: {sum(1 for o in forecasted if o.trace)} " + f"({sum(1 for o in forecasted if o.traces and not o.trace)} ran only after " + "spot scoring time)" + ) + + lines += ["", "### By tournament", ""] + for label, count in Counter(_project_label(o) for o in questions).most_common(): + scored_here = sum(1 for o in scored if _project_label(o) == label) + lines.append(f"- {label}: {count} questions ({scored_here} scored)") + + metric_label = f"{_metric_label(metric)} score" + lines += ["", f"## Best {min(top_n, len(ranked))} (by {metric_label})", ""] + lines += [_question_line(i, o, metric) for i, o in enumerate(ranked[:top_n], 1)] + lines += ["", f"## Worst {min(top_n, len(ranked))} (by {metric_label})", ""] + lines += [ + _question_line(i, o, metric) for i, o in enumerate(reversed(ranked[-top_n:]), 1) + ] + + return "\n".join(lines) + "\n" diff --git a/forecasting_tools/bot_review/traces.py b/forecasting_tools/bot_review/traces.py new file mode 100644 index 00000000..f778c554 --- /dev/null +++ b/forecasting_tools/bot_review/traces.py @@ -0,0 +1,124 @@ +"""Attach the bot's own comments to an outcome table as run traces. + +The comment a bot posts is its report, so it is the trace of that run. A +question can have several, one per run; the one that earned the score is the +run standing when the question was spot scored. +""" + +from __future__ import annotations + +import logging +import re +from collections import defaultdict + +from forecasting_tools.bot_review.outcomes import OutcomeTable, RunTrace +from forecasting_tools.bot_review.report_parsing import ( + forecaster_rationales, + parse_forecasters, + split_sections, + was_truncated, +) +from forecasting_tools.data_models.comment import Comment +from forecasting_tools.helpers.metaculus_client import MetaculusClient + +logger = logging.getLogger(__name__) + +COST_PATTERN = re.compile(r"^\*Total Cost\*:\s*\$([\d.]+)", re.MULTILINE) +MINUTES_PATTERN = re.compile(r"^\*Time Spent\*:\s*([\d.]+) minutes", re.MULTILINE) +QUESTION_PATTERN = re.compile(r"^\*Question\*:(.*)$", re.MULTILINE) + + +def reduce_comment(comment: Comment) -> RunTrace: + """Reduce a posted report to the fields a review needs.""" + summary = split_sections(comment.text).get("summary", "") + question = QUESTION_PATTERN.search(comment.text) + cost = COST_PATTERN.search(comment.text) + minutes = MINUTES_PATTERN.search(comment.text) + return RunTrace( + comment_id=comment.id, + run_time=comment.created_at, + question_text=question.group(1).strip() if question else None, + forecasters=parse_forecasters(summary), + cost=float(cost.group(1)) if cost else None, + minutes=float(minutes.group(1)) if minutes else None, + truncated=was_truncated(comment.text), + ) + + +def attach_traces( + table: OutcomeTable, + client: MetaculusClient | None = None, + max_comments: int = 500, +) -> None: + """ + Fill in ``traces`` on every question in the table. + + Bots publish their reports privately or publicly depending on how they are + configured, so both are fetched. + + :param table: the table to attach to, modified in place + :param client: client to fetch with, created from the environment if not given + :param max_comments: how far back to page, per privacy setting + """ + client = client or MetaculusClient() + user_id = client.get_current_user_id() + comments = [ + comment + for is_private in (True, False) + for comment in client.get_own_comments( + is_private=is_private, user_id=user_id, max_comments=max_comments + ) + ] + rows_by_post = defaultdict(list) + for outcome in table.questions: + rows_by_post[outcome.post_id].append(outcome) + comments_by_post = defaultdict(list) + for comment in comments: + if comment.on_post in rows_by_post: + comments_by_post[comment.on_post].append(comment) + logger.info( + f"Read {len(comments)} comments, {sum(map(len, comments_by_post.values()))} " + f"of them on the {len(rows_by_post)} posts in the table" + ) + for post_id, rows in rows_by_post.items(): + runs = [reduce_comment(c) for c in comments_by_post[post_id]] + for outcome in rows: + outcome.traces = ( + runs + if len(rows) == 1 + else [run for run in runs if run.question_text == outcome.title] + ) + + +def get_trace( + post_id: int, + section: str = "research", + forecaster: str | None = None, + comment_id: int | None = None, + client: MetaculusClient | None = None, +) -> str: + """ + One part of a report the bot posted, by default its latest on the post. + + :param post_id: post id, as it appears in question urls + :param section: summary, research or forecasts + :param forecaster: a key like ``R1:F2``, to read that rationale instead + :param comment_id: a trace's ``comment_id``, to read the run that scored + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + user_id = client.get_current_user_id() + comments = [ + comment + for is_private in (True, False) + for comment in client.get_own_comments( + post_id=post_id, is_private=is_private, user_id=user_id + ) + if comment_id is None or comment.id == comment_id + ] + if not comments: + return "" + text = max(comments, key=lambda comment: comment.created_at).text + if forecaster: + return forecaster_rationales(text).get(forecaster, "") + return split_sections(text).get(section, "") diff --git a/forecasting_tools/data_models/comment.py b/forecasting_tools/data_models/comment.py new file mode 100644 index 00000000..03aa5c93 --- /dev/null +++ b/forecasting_tools/data_models/comment.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel + + +class Comment(BaseModel): + """A comment on a Metaculus post. ``on_post`` is a post id, not a question id.""" + + id: int + on_post: int + author_id: int + author_username: str + created_at: datetime + text: str + is_private: bool + + @classmethod + def from_metaculus_api_json(cls, comment_json: dict) -> Comment: + author = comment_json["author"] + return cls( + id=comment_json["id"], + on_post=comment_json["on_post"], + author_id=author["id"], + author_username=author["username"], + created_at=comment_json["created_at"], + text=comment_json["text"], + is_private=comment_json["is_private"], + ) diff --git a/forecasting_tools/data_models/leaderboard.py b/forecasting_tools/data_models/leaderboard.py new file mode 100644 index 00000000..07c7bf64 --- /dev/null +++ b/forecasting_tools/data_models/leaderboard.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class LeaderboardEntry(BaseModel): + """One forecaster's standing on a leaderboard. + + Community aggregates appear as entries with no user and an + ``aggregation_method`` set. + """ + + user_id: int | None + username: str | None + aggregation_method: str | None + rank: int | None + score: float | None + coverage: float | None + contribution_count: int | None + medal: str | None + prize: float | None + excluded: bool + + @classmethod + def from_metaculus_api_json(cls, entry_json: dict) -> LeaderboardEntry: + user = entry_json.get("user") or {} + return cls( + user_id=user.get("id"), + username=user.get("username"), + aggregation_method=entry_json.get("aggregation_method"), + rank=entry_json["rank"], + score=entry_json["score"], + coverage=entry_json["coverage"], + contribution_count=entry_json["contribution_count"], + medal=entry_json["medal"], + prize=entry_json["prize"], + excluded=entry_json["excluded"], + ) + + +class Leaderboard(BaseModel): + """A project's leaderboard. ``user_entry`` is None if you have no standing on it.""" + + project_id: int + project_name: str | None + project_slug: str | None + score_type: str + finalized: bool + entries: list[LeaderboardEntry] + user_entry: LeaderboardEntry | None + + @classmethod + def from_metaculus_api_json(cls, leaderboard_json: dict) -> Leaderboard: + user_entry = leaderboard_json.get("userEntry") + return cls( + project_id=leaderboard_json["project_id"], + project_name=leaderboard_json.get("project_name"), + project_slug=leaderboard_json.get("project_slug"), + score_type=leaderboard_json["score_type"], + finalized=leaderboard_json["finalized"], + entries=[ + LeaderboardEntry.from_metaculus_api_json(entry) + for entry in leaderboard_json.get("entries") or [] + ], + user_entry=( + LeaderboardEntry.from_metaculus_api_json(user_entry) + if user_entry + else None + ), + ) diff --git a/forecasting_tools/helpers/metaculus_client.py b/forecasting_tools/helpers/metaculus_client.py index aeedfff1..7319ce4a 100644 --- a/forecasting_tools/helpers/metaculus_client.py +++ b/forecasting_tools/helpers/metaculus_client.py @@ -23,7 +23,9 @@ DetailedCoherenceLink, NeedsUpdateResponse, ) +from forecasting_tools.data_models.comment import Comment from forecasting_tools.data_models.data_organizer import DataOrganizer +from forecasting_tools.data_models.leaderboard import Leaderboard from forecasting_tools.data_models.questions import ( BinaryQuestion, ConditionalQuestion, @@ -154,6 +156,7 @@ class MetaculusClient: ) # Going forward, please use the https://metaculus.com/tournament/bot-testing-area/ questions for testing MAX_QUESTIONS_FROM_QUESTION_API_PER_REQUEST = 100 + MAX_COMMENTS_FROM_COMMENT_API_PER_REQUEST = 100 def __init__( self, @@ -489,6 +492,81 @@ def get_current_user_id(self) -> int: content = json.loads(response.content) return int(content["id"]) + def get_own_comments( + self, + post_id: int | None = None, + is_private: bool = True, + user_id: int | None = None, + max_comments: int = 500, + ) -> list[Comment]: + """ + Comments you posted, newest first. The API only allows listing your own. + + :param post_id: only return comments on this post + :param is_private: whether to return private or public comments + :param user_id: your own user id, looked up if not given + :param max_comments: stop paging once this many are collected + """ + author_id = user_id if user_id is not None else self.get_current_user_id() + comments: list[Comment] = [] + while len(comments) < max_comments: + page_size = min( + self.MAX_COMMENTS_FROM_COMMENT_API_PER_REQUEST, + max_comments - len(comments), + ) + params: dict[str, Any] = { + "author": author_id, + "is_private": str(is_private).lower(), + "limit": page_size, + "offset": len(comments), + } + if post_id is not None: + params["post"] = post_id + page = self._get_comment_page(params) + comments.extend(Comment.from_metaculus_api_json(item) for item in page) + if len(page) < page_size: + break + logger.info(f"Retrieved {len(comments)} comments for user {author_id}") + return comments + + @retry_with_exponential_backoff() + def _get_comment_page(self, params: dict[str, Any]) -> list[dict]: + self._sleep_between_requests() + response = requests.get( + f"{self.base_url}/comments/", + params=params, + **self._get_auth_headers(), # type: ignore + timeout=self.timeout, + ) + raise_for_status_with_additional_info(response) + return json.loads(response.content)["results"] + + @retry_with_exponential_backoff() + def get_project_leaderboard(self, project_id: int) -> Leaderboard: + """ + The primary leaderboard for a project, including your own entry. + + :param project_id: numeric project id (slugs are not accepted here) + """ + self._sleep_between_requests() + response = requests.get( + f"{self.base_url}/leaderboards/project/{project_id}/", + params={"with_entries": "true"}, + **self._get_auth_headers(), # type: ignore + timeout=self.timeout, + ) + raise_for_status_with_additional_info(response) + leaderboards = json.loads(response.content) + primary = next( + leaderboard + for leaderboard in leaderboards + if leaderboard["is_primary_leaderboard"] + ) + logger.info( + f"Retrieved leaderboard for project {project_id} with {len(primary.get('entries') or [])} entries" + ) + return Leaderboard.from_metaculus_api_json(primary) + @retry_with_exponential_backoff() def post_question_link( self, diff --git a/pyproject.toml b/pyproject.toml index c8b322f6..afe6e6a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,7 @@ source-archive = ["boto3", "playwright", "firecrawl-py", "trafilatura", "pymupdf [tool.poetry.scripts] source-archive = "forecasting_tools.agents_and_tools.source_archive.cli:main" +bot-review = "forecasting_tools.bot_review.cli:main" [tool.poetry.group.dev.dependencies] time-machine = ">=2.19.0,<4.0.0"