From f4524ead90afc69296b57839dbffbd6f040bb1c1 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:08:40 +0100 Subject: [PATCH 01/21] Add report explanation parser for bot review Reads the structure back out of the explanation string that ForecastBot._create_comment produces: the three top level sections, each forecaster's prediction from the summary bullets, and each forecaster's rationale. The same string is saved to disk and posted as a Metaculus comment, so this reads either source. Forecasters are keyed R:F because a bot with research_reports_per_question > 1 has a forecaster 1 in every report. Model names on the bullets are optional; the framework does not write them, though bots may annotate their own. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_report_parsing.py | 183 ++++++++++++++++++ forecasting_tools/bot_review/__init__.py | 1 + .../bot_review/report_parsing.py | 103 ++++++++++ 3 files changed, 287 insertions(+) create mode 100644 code_tests/unit_tests/test_bot_review/test_report_parsing.py create mode 100644 forecasting_tools/bot_review/__init__.py create mode 100644 forecasting_tools/bot_review/report_parsing.py diff --git a/code_tests/unit_tests/test_bot_review/test_report_parsing.py b/code_tests/unit_tests/test_bot_review/test_report_parsing.py new file mode 100644 index 00000000..059d549e --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_report_parsing.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from forecasting_tools.bot_review.report_parsing import ( + final_prediction, + forecaster_rationales, + parse_forecasters, + split_sections, + was_truncated, +) + +# Stock forecasting-tools output: unannotated bullets, two research reports. +TEMPLATE_EXPLANATION = """ +# SUMMARY +*Question*: Will X happen? +*Final Prediction*: 35.0% +*Total Cost*: $0.35 (estimated) + +## Report 1 Summary +### Forecasts +*Forecaster 1*: 30.0% +*Forecaster 2*: 40.0% + +### Research Summary +A short summary of report 1. + +## Report 2 Summary +### Forecasts +*Forecaster 1*: 50.0% +*Forecaster 2*: 60.0% + +### Research Summary +A short summary of report 2. + +# RESEARCH +## Report 1 Research +### Research heading 1 +Body one. +## Report 2 Research +### Research heading 2 +Body two. + +# FORECASTS + +## R1: Forecaster 1 Reasoning +### A heading demoted by the framework +R1F1 reasoning. + +## R1: Forecaster 2 Reasoning +R1F2 reasoning. + +## R2: Forecaster 1 Reasoning +R2F1 reasoning. + +## R2: Forecaster 2 Reasoning +R2F2 reasoning. +""".strip() + +# A bot that annotates its bullets with model names, and a model that emitted a +# top level heading inside its rationale. +ANNOTATED_EXPLANATION = """ +# SUMMARY +*Question*: Will X happen? +*Final Prediction*: 3.0% + +## Report 1 Summary +### Forecasts +*Forecaster 1 (gpt-5.4)*: 3.0% +*Forecaster 2 (gpt-5.5)*: 4.0% + +### Research Summary +_Full research in the RESEARCH section below._ + +# RESEARCH +## Report 1 Research +Some news. + +# FORECASTS +## R1: Forecaster 1 Reasoning +First rationale. + +# Analysis: a heading the model emitted mid rationale +Still forecaster 1. + +## R1: Forecaster 2 Reasoning +Second rationale. +""".strip() + +MULTILINE_SUMMARY = """ +*Final Prediction*: +- A: 60.00% +- B: 40.00% + +*Total Cost*: disabled + +## Report 1 Summary +### Forecasts +*Forecaster 1*: - A: 62.0% +- B: 38.0% + +*Forecaster 2*: - A: 58.0% +- B: 42.0% + +### Research Summary +_elsewhere_ +""".strip() + + +class TestSplitSections: + def test_finds_the_three_sections(self): + assert set(split_sections(TEMPLATE_EXPLANATION)) == { + "summary", + "research", + "forecasts", + } + + def test_research_stops_where_forecasts_start(self): + research = split_sections(TEMPLATE_EXPLANATION)["research"] + assert "Body two." in research + assert "Forecaster 1 Reasoning" not in research + + def test_missing_sections_are_omitted(self): + assert split_sections("# SUMMARY\nonly this") == {"summary": "only this"} + + +class TestParseForecasters: + def test_unannotated_bullets_across_two_reports(self): + summary = split_sections(TEMPLATE_EXPLANATION)["summary"] + assert parse_forecasters(summary) == [ + {"key": "R1:F1", "model": None, "prediction": "30.0%"}, + {"key": "R1:F2", "model": None, "prediction": "40.0%"}, + {"key": "R2:F1", "model": None, "prediction": "50.0%"}, + {"key": "R2:F2", "model": None, "prediction": "60.0%"}, + ] + + def test_model_names_when_the_bot_annotates_them(self): + summary = split_sections(ANNOTATED_EXPLANATION)["summary"] + assert parse_forecasters(summary) == [ + {"key": "R1:F1", "model": "gpt-5.4", "prediction": "3.0%"}, + {"key": "R1:F2", "model": "gpt-5.5", "prediction": "4.0%"}, + ] + + def test_research_summary_is_not_read_as_a_prediction(self): + summary = split_sections(ANNOTATED_EXPLANATION)["summary"] + assert "Full research" not in parse_forecasters(summary)[-1]["prediction"] + + def test_multiline_predictions_are_kept_whole(self): + parsed = parse_forecasters(MULTILINE_SUMMARY) + assert parsed[0]["prediction"] == "- A: 62.0%\n- B: 38.0%" + assert parsed[1]["prediction"] == "- A: 58.0%\n- B: 42.0%" + + +class TestForecasterRationales: + def test_second_research_report_does_not_overwrite_the_first(self): + rationales = forecaster_rationales(TEMPLATE_EXPLANATION) + assert sorted(rationales) == ["R1:F1", "R1:F2", "R2:F1", "R2:F2"] + assert "R1F1 reasoning." in rationales["R1:F1"] + assert "R2F1 reasoning." in rationales["R2:F1"] + + def test_heading_inside_a_rationale_does_not_end_it(self): + rationales = forecaster_rationales(ANNOTATED_EXPLANATION) + assert sorted(rationales) == ["R1:F1", "R1:F2"] + assert "Still forecaster 1." in rationales["R1:F1"] + assert "Second rationale" not in rationales["R1:F1"] + + +class TestFinalPrediction: + def test_single_line(self): + summary = split_sections(TEMPLATE_EXPLANATION)["summary"] + assert final_prediction(summary) == "35.0%" + + def test_multiline(self): + assert final_prediction(MULTILINE_SUMMARY) == "- A: 60.00%\n- B: 40.00%" + + def test_absent(self): + assert final_prediction("# SUMMARY\nnothing here") == "" + + +class TestWasTruncated: + def test_detects_the_frameworks_truncation_notice(self): + assert was_truncated( + "# SUMMARY\n---\n The comment size exceeded max size and has been truncated" + ) + assert not was_truncated(TEMPLATE_EXPLANATION) diff --git a/forecasting_tools/bot_review/__init__.py b/forecasting_tools/bot_review/__init__.py new file mode 100644 index 00000000..640ec129 --- /dev/null +++ b/forecasting_tools/bot_review/__init__.py @@ -0,0 +1 @@ +"""Tools for reviewing how a bot performed on the questions it forecast.""" diff --git a/forecasting_tools/bot_review/report_parsing.py b/forecasting_tools/bot_review/report_parsing.py new file mode 100644 index 00000000..9148be17 --- /dev/null +++ b/forecasting_tools/bot_review/report_parsing.py @@ -0,0 +1,103 @@ +"""Pull structure back out of a report explanation. + +``ForecastBot._create_comment`` assembles every report into the same shape: a +SUMMARY listing each forecaster's prediction, the RESEARCH behind them, and one +FORECASTS subsection per forecaster. That string is what gets saved to disk and +what gets posted to Metaculus as a comment, so the same functions read either. + +Forecasters are keyed ``R:F``, since a bot running +more than one research report per question has a forecaster 1 in each. +""" + +from __future__ import annotations + +import re + +SECTIONS = ("summary", "research", "forecasts") +REPORT_PATTERN = re.compile(r"^## Report (\d+) Summary\s*$", re.MULTILINE) +FORECASTER_PATTERN = re.compile( + r"^\*Forecaster (\d+)(?: \(([^)]*)\))?\*:", re.MULTILINE +) +RATIONALE_PATTERN = re.compile( + r"^## R(\d+): Forecaster (\d+) Reasoning\s*$", re.MULTILINE +) +FINAL_PREDICTION_PATTERN = re.compile( + r"^\*Final Prediction\*:(.*?)(?=^\*[A-Z]|^## |\Z)", re.MULTILINE | re.DOTALL +) + + +def split_sections(explanation: str) -> dict[str, str]: + """The SUMMARY / RESEARCH / FORECASTS blocks, by lowercased name.""" + marks = [] + for name in SECTIONS: + match = re.search(rf"^# {name.upper()}\s*$", explanation, re.MULTILINE) + if match: + marks.append((match.start(), match.end(), name)) + marks.sort() + sections = {} + for i, (_, end, name) in enumerate(marks): + stop = marks[i + 1][0] if i + 1 < len(marks) else len(explanation) + sections[name] = explanation[end:stop].strip() + return sections + + +def _bodies_between(text: str, marks: list[tuple[int, int]]) -> list[str]: + return [ + text[end : (marks[i + 1][0] if i + 1 < len(marks) else len(text))].strip() + for i, (_, end) in enumerate(marks) + ] + + +def forecasts_block(summary: str) -> str: + """The ``### Forecasts`` block of one report's summary.""" + match = re.search(r"^### Forecasts\s*$", summary, re.MULTILINE) + if not match: + return "" + rest = summary[match.end() :] + following = re.search(r"^### ", rest, re.MULTILINE) + return rest[: following.start()] if following else rest + + +def parse_forecasters(summary: str) -> list[dict]: + """Each forecaster's prediction as it was written in the summary. + + ``model`` is None unless the bot annotates its own bullets with model names; + stock forecasting-tools does not. + """ + reports = list(REPORT_PATTERN.finditer(summary)) + blocks = _bodies_between(summary, [(m.start(), m.end()) for m in reports]) + forecasters = [] + for report, block in zip(reports, blocks): + bullets = forecasts_block(block) + matches = list(FORECASTER_PATTERN.finditer(bullets)) + bodies = _bodies_between(bullets, [(m.start(), m.end()) for m in matches]) + forecasters += [ + { + "key": f"R{report.group(1)}:F{match.group(1)}", + "model": match.group(2), + "prediction": body, + } + for match, body in zip(matches, bodies) + ] + return forecasters + + +def forecaster_rationales(explanation: str) -> dict[str, str]: + """Full reasoning text for each forecaster.""" + block = split_sections(explanation).get("forecasts", "") + matches = list(RATIONALE_PATTERN.finditer(block)) + bodies = _bodies_between(block, [(m.start(), m.end()) for m in matches]) + return { + f"R{match.group(1)}:F{match.group(2)}": body + for match, body in zip(matches, bodies) + } + + +def final_prediction(summary: str) -> str: + match = FINAL_PREDICTION_PATTERN.search(summary) + return match.group(1).strip() if match else "" + + +def was_truncated(explanation: str) -> bool: + """Whether the explanation hit the comment size cap and lost its tail.""" + return "has been truncated" in explanation From f99cf0bd64075ef79805f1d59bb7b4d3375d0418 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 21:14:27 +0100 Subject: [PATCH 02/21] Use MarkdownTree for section splitting Sections and forecaster rationales come from the same splitter ForecastReport already uses, rather than a second regex implementation. Only the summary bullets are read directly, since they are not headings. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_report_parsing.py | 31 +++++------ .../bot_review/report_parsing.py | 54 +++++++++---------- 2 files changed, 43 insertions(+), 42 deletions(-) diff --git a/code_tests/unit_tests/test_bot_review/test_report_parsing.py b/code_tests/unit_tests/test_bot_review/test_report_parsing.py index 059d549e..5212c002 100644 --- a/code_tests/unit_tests/test_bot_review/test_report_parsing.py +++ b/code_tests/unit_tests/test_bot_review/test_report_parsing.py @@ -55,8 +55,7 @@ R2F2 reasoning. """.strip() -# A bot that annotates its bullets with model names, and a model that emitted a -# top level heading inside its rationale. +# Bots may annotate the bullets with the model behind each forecaster. ANNOTATED_EXPLANATION = """ # SUMMARY *Question*: Will X happen? @@ -64,8 +63,8 @@ ## Report 1 Summary ### Forecasts -*Forecaster 1 (gpt-5.4)*: 3.0% -*Forecaster 2 (gpt-5.5)*: 4.0% +*Forecaster 1 (a-model)*: 3.0% +*Forecaster 2 (another-model)*: 4.0% ### Research Summary _Full research in the RESEARCH section below._ @@ -78,9 +77,6 @@ ## R1: Forecaster 1 Reasoning First rationale. -# Analysis: a heading the model emitted mid rationale -Still forecaster 1. - ## R1: Forecaster 2 Reasoning Second rationale. """.strip() @@ -119,7 +115,10 @@ def test_research_stops_where_forecasts_start(self): assert "Forecaster 1 Reasoning" not in research def test_missing_sections_are_omitted(self): - assert split_sections("# SUMMARY\nonly this") == {"summary": "only this"} + # the heading line stays in, as it does on ForecastReport.summary + assert split_sections("# SUMMARY\nonly this") == { + "summary": "# SUMMARY\nonly this" + } class TestParseForecasters: @@ -135,8 +134,8 @@ def test_unannotated_bullets_across_two_reports(self): def test_model_names_when_the_bot_annotates_them(self): summary = split_sections(ANNOTATED_EXPLANATION)["summary"] assert parse_forecasters(summary) == [ - {"key": "R1:F1", "model": "gpt-5.4", "prediction": "3.0%"}, - {"key": "R1:F2", "model": "gpt-5.5", "prediction": "4.0%"}, + {"key": "R1:F1", "model": "a-model", "prediction": "3.0%"}, + {"key": "R1:F2", "model": "another-model", "prediction": "4.0%"}, ] def test_research_summary_is_not_read_as_a_prediction(self): @@ -156,11 +155,13 @@ def test_second_research_report_does_not_overwrite_the_first(self): assert "R1F1 reasoning." in rationales["R1:F1"] assert "R2F1 reasoning." in rationales["R2:F1"] - def test_heading_inside_a_rationale_does_not_end_it(self): - rationales = forecaster_rationales(ANNOTATED_EXPLANATION) - assert sorted(rationales) == ["R1:F1", "R1:F2"] - assert "Still forecaster 1." in rationales["R1:F1"] - assert "Second rationale" not in rationales["R1:F1"] + def test_subheadings_stay_with_their_forecaster(self): + rationales = forecaster_rationales(TEMPLATE_EXPLANATION) + assert "A heading demoted by the framework" in rationales["R1:F1"] + assert "R1F2 reasoning" not in rationales["R1:F1"] + + def test_no_forecasts_section(self): + assert forecaster_rationales("# SUMMARY\nnothing else") == {} class TestFinalPrediction: diff --git a/forecasting_tools/bot_review/report_parsing.py b/forecasting_tools/bot_review/report_parsing.py index 9148be17..851d7b60 100644 --- a/forecasting_tools/bot_review/report_parsing.py +++ b/forecasting_tools/bot_review/report_parsing.py @@ -5,6 +5,9 @@ FORECASTS subsection per forecaster. That string is what gets saved to disk and what gets posted to Metaculus as a comment, so the same functions read either. +Sections come from ``MarkdownTree``, the same splitter ``ForecastReport`` uses. +The summary bullets are not headings, so they are read directly. + Forecasters are keyed ``R:F``, since a bot running more than one research report per question has a forecaster 1 in each. """ @@ -13,14 +16,14 @@ import re +from forecasting_tools.data_models.markdown_tree import MarkdownTree + SECTIONS = ("summary", "research", "forecasts") REPORT_PATTERN = re.compile(r"^## Report (\d+) Summary\s*$", re.MULTILINE) FORECASTER_PATTERN = re.compile( r"^\*Forecaster (\d+)(?: \(([^)]*)\))?\*:", re.MULTILINE ) -RATIONALE_PATTERN = re.compile( - r"^## R(\d+): Forecaster (\d+) Reasoning\s*$", re.MULTILINE -) +RATIONALE_TITLE_PATTERN = re.compile(r"^R(\d+): Forecaster (\d+) Reasoning$") FINAL_PREDICTION_PATTERN = re.compile( r"^\*Final Prediction\*:(.*?)(?=^\*[A-Z]|^## |\Z)", re.MULTILINE | re.DOTALL ) @@ -28,17 +31,25 @@ def split_sections(explanation: str) -> dict[str, str]: """The SUMMARY / RESEARCH / FORECASTS blocks, by lowercased name.""" - marks = [] - for name in SECTIONS: - match = re.search(rf"^# {name.upper()}\s*$", explanation, re.MULTILINE) - if match: - marks.append((match.start(), match.end(), name)) - marks.sort() - sections = {} - for i, (_, end, name) in enumerate(marks): - stop = marks[i + 1][0] if i + 1 < len(marks) else len(explanation) - sections[name] = explanation[end:stop].strip() - return sections + return { + section.title.strip().lower(): section.text_of_section_and_subsections.strip() + for section in MarkdownTree.turn_markdown_into_report_sections(explanation) + if section.title and section.title.strip().lower() in SECTIONS + } + + +def forecaster_rationales(explanation: str) -> dict[str, str]: + """Full reasoning text for each forecaster.""" + rationales = {} + for section in MarkdownTree.turn_markdown_into_report_sections(explanation): + if not section.title or section.title.strip().lower() != "forecasts": + continue + for subsection in section.sub_sections: + match = RATIONALE_TITLE_PATTERN.match((subsection.title or "").strip()) + if match: + key = f"R{match.group(1)}:F{match.group(2)}" + rationales[key] = subsection.text_of_section_and_subsections.strip() + return rationales def _bodies_between(text: str, marks: list[tuple[int, int]]) -> list[str]: @@ -61,8 +72,8 @@ def forecasts_block(summary: str) -> str: def parse_forecasters(summary: str) -> list[dict]: """Each forecaster's prediction as it was written in the summary. - ``model`` is None unless the bot annotates its own bullets with model names; - stock forecasting-tools does not. + ``model`` is None unless the bot annotates its own bullets with model names, + which the framework does not do. """ reports = list(REPORT_PATTERN.finditer(summary)) blocks = _bodies_between(summary, [(m.start(), m.end()) for m in reports]) @@ -82,17 +93,6 @@ def parse_forecasters(summary: str) -> list[dict]: return forecasters -def forecaster_rationales(explanation: str) -> dict[str, str]: - """Full reasoning text for each forecaster.""" - block = split_sections(explanation).get("forecasts", "") - matches = list(RATIONALE_PATTERN.finditer(block)) - bodies = _bodies_between(block, [(m.start(), m.end()) for m in matches]) - return { - f"R{match.group(1)}:F{match.group(2)}": body - for match, body in zip(matches, bodies) - } - - def final_prediction(summary: str) -> str: match = FINAL_PREDICTION_PATTERN.search(summary) return match.group(1).strip() if match else "" From 80225f0d6803a4a7dd5568646215c7270d8c0edc Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:11:55 +0100 Subject: [PATCH 03/21] Add own-comment and project-leaderboard fetches to MetaculusClient The bot review needs two things the client could not yet fetch: the comments a bot posted (its forecast reports, since the comment text is the report explanation verbatim) and the tournament standing they earned. get_own_comments defaults to is_private=True. post_question_comment posts private comments by default, and the list endpoint omits private comments unless asked for them, so a bot's own reports are otherwise invisible -- and the API returns no count, making that indistinguishable from a bot that never commented. The endpoint also refuses any author but your own, so there is no cross-bot variant to offer. Leaderboard entries include community aggregates, which have no user and carry an aggregation_method instead. They are ranked alongside forecasters, so they are kept rather than filtered: comparing your rank against them is how you tell whether you beat the crowd. Verified live against the Metaculus API: three freshly posted comments round-trip, and the MiniBench 2026-06-29 leaderboard parses to the same rank and score the site reports. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_client_calls.py | 150 ++++++++++++++++++ forecasting_tools/data_models/comment.py | 39 +++++ forecasting_tools/data_models/leaderboard.py | 75 +++++++++ forecasting_tools/helpers/metaculus_client.py | 86 ++++++++++ 4 files changed, 350 insertions(+) create mode 100644 code_tests/unit_tests/test_bot_review/test_client_calls.py create mode 100644 forecasting_tools/data_models/comment.py create mode 100644 forecasting_tools/data_models/leaderboard.py diff --git a/code_tests/unit_tests/test_bot_review/test_client_calls.py b/code_tests/unit_tests/test_bot_review/test_client_calls.py new file mode 100644 index 00000000..14389b21 --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_client_calls.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +from unittest.mock import patch + +from forecasting_tools.data_models.comment import Comment +from forecasting_tools.data_models.leaderboard import Leaderboard +from forecasting_tools.helpers.metaculus_client import MetaculusClient + +# Field names and nesting copied from live API responses on 2026-07-24. +COMMENT_JSON = { + "id": 977364, + "on_post": 44676, + "author": {"id": 305884, "username": "a-bot", "is_bot": True}, + "created_at": "2026-07-24T20:49:49.595398Z", + "text": "# SUMMARY\n*Final Prediction*: 20.0%", + "is_private": True, + "included_forecast": {"start_time": "2026-07-24T20:49:44.998152Z"}, + "parent_id": None, +} + +ENTRY_JSON = { + "user": {"id": 303267, "username": "a-bot", "is_bot": True}, + "rank": 17, + "score": 348.50171781642365, + "coverage": 36, + "contribution_count": 36, + "medal": None, + "prize": 0, + "take": 121453.44732099818, + "excluded": False, +} + +LEADERBOARD_JSON = { + "id": 932, + "is_primary_leaderboard": True, + "project_id": 33064, + "project_name": "MiniBench - 2026-06-29", + "project_slug": "minibench-2026-06-29", + "project_type": "question_series", + "score_type": "spot_peer_tournament", + "finalized": True, + "entries": [ENTRY_JSON], + "userEntry": ENTRY_JSON, +} + + +class TestCommentParsing: + def test_reads_the_live_field_names(self): + comment = Comment.from_metaculus_api_json(COMMENT_JSON) + assert comment.id == 977364 + assert comment.on_post == 44676 + assert comment.author_id == 305884 + assert comment.author_username == "a-bot" + assert comment.is_private + assert comment.text.startswith("# SUMMARY") + assert comment.created_at.year == 2026 + + +class TestLeaderboardParsing: + def test_entry_fields(self): + leaderboard = Leaderboard.from_metaculus_api_json(LEADERBOARD_JSON) + assert leaderboard.project_id == 33064 + assert leaderboard.score_type == "spot_peer_tournament" + assert leaderboard.finalized + assert len(leaderboard.entries) == 1 + assert leaderboard.user_entry is not None + assert leaderboard.user_entry.rank == 17 + assert leaderboard.user_entry.user_id == 303267 + assert not leaderboard.user_entry.excluded + + def test_no_entry_when_you_never_forecast_the_tournament(self): + without_entry = {**LEADERBOARD_JSON, "userEntry": None} + assert Leaderboard.from_metaculus_api_json(without_entry).user_entry is None + + def test_community_aggregates_rank_alongside_forecasters(self): + # These entries have no user; they are the crowd you are being compared to. + aggregate = { + **ENTRY_JSON, + "user": None, + "aggregation_method": "recency_weighted", + "rank": 16, + } + leaderboard = Leaderboard.from_metaculus_api_json( + {**LEADERBOARD_JSON, "entries": [ENTRY_JSON, aggregate]} + ) + crowd = leaderboard.entries[1] + assert crowd.user_id is None + assert crowd.username is None + assert crowd.aggregation_method == "recency_weighted" + assert leaderboard.entries[0].aggregation_method is None + + def test_a_medalled_entry_keeps_its_medal(self): + medalled = { + **LEADERBOARD_JSON, + "userEntry": {**ENTRY_JSON, "medal": "gold", "rank": 1}, + } + entry = Leaderboard.from_metaculus_api_json(medalled).user_entry + assert entry is not None and entry.medal == "gold" + + +class TestGetOwnComments: + def client(self) -> MetaculusClient: + return MetaculusClient(token="a-token", sleep_seconds_between_requests=0) + + def test_asks_for_private_comments_by_default(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[] + ) as page: + self.client().get_own_comments(user_id=305884) + assert page.call_args.args[0]["is_private"] == "true" + assert page.call_args.args[0]["author"] == 305884 + + def test_post_filter_is_only_sent_when_given(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[] + ) as page: + self.client().get_own_comments(user_id=1) + assert "post" not in page.call_args.args[0] + self.client().get_own_comments(user_id=1, post_id=44676) + assert page.call_args.args[0]["post"] == 44676 + + def test_supplied_user_id_avoids_the_lookup_request(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[] + ), patch.object(MetaculusClient, "get_current_user_id") as lookup: + self.client().get_own_comments(user_id=305884) + lookup.assert_not_called() + + def test_pages_until_a_short_page_arrives(self): + full_page = [ + COMMENT_JSON + ] * MetaculusClient.MAX_COMMENTS_FROM_COMMENT_API_PER_REQUEST + with patch.object( + MetaculusClient, + "_get_comment_page", + side_effect=[full_page, [COMMENT_JSON]], + ) as page: + comments = self.client().get_own_comments(user_id=1) + assert len(comments) == 101 + assert page.call_count == 2 + assert page.call_args_list[1].args[0]["offset"] == 100 + + def test_stops_at_max_comments_without_overfetching(self): + with patch.object( + MetaculusClient, "_get_comment_page", return_value=[COMMENT_JSON] * 5 + ) as page: + comments = self.client().get_own_comments(user_id=1, max_comments=5) + assert len(comments) == 5 + assert page.call_args.args[0]["limit"] == 5 + assert page.call_count == 1 diff --git a/forecasting_tools/data_models/comment.py b/forecasting_tools/data_models/comment.py new file mode 100644 index 00000000..08289830 --- /dev/null +++ b/forecasting_tools/data_models/comment.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from datetime import datetime + +from pydantic import BaseModel + + +class Comment(BaseModel): + """A comment on a Metaculus post. + + A bot publishing a report posts the report's explanation as the comment + text, so ``text`` parses with the same functions that read a saved + ``ForecastReport``. + + ``on_post`` is a post id, not a question id. The two are separate sequences + that overlap, so joining question-level data on ``on_post`` silently matches + the wrong question rather than matching nothing. + """ + + id: int + on_post: int + author_id: int + author_username: str + created_at: datetime + text: str + is_private: bool + + @classmethod + def from_metaculus_api_json(cls, comment_json: dict) -> Comment: + author = comment_json["author"] + return cls( + id=comment_json["id"], + on_post=comment_json["on_post"], + author_id=author["id"], + author_username=author["username"], + created_at=comment_json["created_at"], + text=comment_json["text"], + is_private=comment_json["is_private"], + ) diff --git a/forecasting_tools/data_models/leaderboard.py b/forecasting_tools/data_models/leaderboard.py new file mode 100644 index 00000000..59ec06c4 --- /dev/null +++ b/forecasting_tools/data_models/leaderboard.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class LeaderboardEntry(BaseModel): + """One forecaster's standing on a leaderboard. + + Community aggregates are ranked alongside forecasters. Those entries have no + user and carry an ``aggregation_method`` instead, so comparing your rank + against them is how you tell whether you beat the crowd. + """ + + user_id: int | None + username: str | None + aggregation_method: str | None + rank: int | None + score: float | None + coverage: float | None + contribution_count: int | None + medal: str | None + prize: float | None + excluded: bool + + @classmethod + def from_metaculus_api_json(cls, entry_json: dict) -> LeaderboardEntry: + user = entry_json.get("user") or {} + return cls( + user_id=user.get("id"), + username=user.get("username"), + aggregation_method=entry_json.get("aggregation_method"), + rank=entry_json["rank"], + score=entry_json["score"], + coverage=entry_json["coverage"], + contribution_count=entry_json["contribution_count"], + medal=entry_json["medal"], + prize=entry_json["prize"], + excluded=entry_json["excluded"], + ) + + +class Leaderboard(BaseModel): + """A tournament's leaderboard, plus your own entry on it. + + ``user_entry`` is None when the requesting user has no standing on this + leaderboard, which is also the case for a tournament they never forecast. + """ + + project_id: int + project_name: str | None + project_slug: str | None + score_type: str + finalized: bool + entries: list[LeaderboardEntry] + user_entry: LeaderboardEntry | None + + @classmethod + def from_metaculus_api_json(cls, leaderboard_json: dict) -> Leaderboard: + user_entry = leaderboard_json.get("userEntry") + return cls( + project_id=leaderboard_json["project_id"], + project_name=leaderboard_json.get("project_name"), + project_slug=leaderboard_json.get("project_slug"), + score_type=leaderboard_json["score_type"], + finalized=leaderboard_json["finalized"], + entries=[ + LeaderboardEntry.from_metaculus_api_json(entry) + for entry in leaderboard_json.get("entries") or [] + ], + user_entry=( + LeaderboardEntry.from_metaculus_api_json(user_entry) + if user_entry + else None + ), + ) diff --git a/forecasting_tools/helpers/metaculus_client.py b/forecasting_tools/helpers/metaculus_client.py index aeedfff1..a888ffcf 100644 --- a/forecasting_tools/helpers/metaculus_client.py +++ b/forecasting_tools/helpers/metaculus_client.py @@ -23,7 +23,9 @@ DetailedCoherenceLink, NeedsUpdateResponse, ) +from forecasting_tools.data_models.comment import Comment from forecasting_tools.data_models.data_organizer import DataOrganizer +from forecasting_tools.data_models.leaderboard import Leaderboard from forecasting_tools.data_models.questions import ( BinaryQuestion, ConditionalQuestion, @@ -154,6 +156,7 @@ class MetaculusClient: ) # Going forward, please use the https://metaculus.com/tournament/bot-testing-area/ questions for testing MAX_QUESTIONS_FROM_QUESTION_API_PER_REQUEST = 100 + MAX_COMMENTS_FROM_COMMENT_API_PER_REQUEST = 100 def __init__( self, @@ -489,6 +492,89 @@ def get_current_user_id(self) -> int: content = json.loads(response.content) return int(content["id"]) + def get_own_comments( + self, + post_id: int | None = None, + is_private: bool = True, + user_id: int | None = None, + max_comments: int = 500, + ) -> list[Comment]: + """ + Comments you posted, newest first. + + The API refuses to list any other author's comments, so this is + deliberately limited to your own. + + ``is_private`` defaults to True because ``post_question_comment`` posts + private comments by default, and the endpoint omits private comments + unless asked for them. A bot's own reports are therefore invisible + without this, and an empty result is indistinguishable from one where + the bot never commented. + + Pass ``user_id`` to skip the lookup request when calling this in a loop. + """ + author_id = user_id if user_id is not None else self.get_current_user_id() + comments: list[Comment] = [] + while len(comments) < max_comments: + page_size = min( + self.MAX_COMMENTS_FROM_COMMENT_API_PER_REQUEST, + max_comments - len(comments), + ) + params: dict[str, Any] = { + "author": author_id, + "is_private": str(is_private).lower(), + "limit": page_size, + "offset": len(comments), + } + if post_id is not None: + params["post"] = post_id + page = self._get_comment_page(params) + comments.extend(Comment.from_metaculus_api_json(item) for item in page) + if len(page) < page_size: + break + logger.info(f"Retrieved {len(comments)} comments for user {author_id}") + return comments + + @retry_with_exponential_backoff() + def _get_comment_page(self, params: dict[str, Any]) -> list[dict]: + self._sleep_between_requests() + response = requests.get( + f"{self.base_url}/comments/", + params=params, + **self._get_auth_headers(), # type: ignore + timeout=self.timeout, + ) + raise_for_status_with_additional_info(response) + return json.loads(response.content)["results"] + + @retry_with_exponential_backoff() + def get_project_leaderboard(self, project_id: int) -> Leaderboard: + """ + The primary leaderboard for a tournament, including your own entry. + + Takes the numeric project id; the slug that works elsewhere 404s here. + The endpoint returns every leaderboard attached to the project, of which + the primary one is the tournament's actual standings. + """ + self._sleep_between_requests() + response = requests.get( + f"{self.base_url}/leaderboards/project/{project_id}/", + params={"with_entries": "true"}, + **self._get_auth_headers(), # type: ignore + timeout=self.timeout, + ) + raise_for_status_with_additional_info(response) + leaderboards = json.loads(response.content) + primary = next( + leaderboard + for leaderboard in leaderboards + if leaderboard["is_primary_leaderboard"] + ) + logger.info( + f"Retrieved leaderboard for project {project_id} with {len(primary.get('entries') or [])} entries" + ) + return Leaderboard.from_metaculus_api_json(primary) + @retry_with_exponential_backoff() def post_question_link( self, From 99de27b692adde4a477a74e2145e462a018f85c1 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:14:35 +0100 Subject: [PATCH 04/21] Trim rationale out of bot review docstrings Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_client_calls.py | 1 - .../bot_review/report_parsing.py | 17 +++++--------- forecasting_tools/data_models/comment.py | 11 +--------- forecasting_tools/data_models/leaderboard.py | 11 +++------- forecasting_tools/helpers/metaculus_client.py | 22 ++++++------------- 5 files changed, 17 insertions(+), 45 deletions(-) diff --git a/code_tests/unit_tests/test_bot_review/test_client_calls.py b/code_tests/unit_tests/test_bot_review/test_client_calls.py index 14389b21..e92bf720 100644 --- a/code_tests/unit_tests/test_bot_review/test_client_calls.py +++ b/code_tests/unit_tests/test_bot_review/test_client_calls.py @@ -73,7 +73,6 @@ def test_no_entry_when_you_never_forecast_the_tournament(self): assert Leaderboard.from_metaculus_api_json(without_entry).user_entry is None def test_community_aggregates_rank_alongside_forecasters(self): - # These entries have no user; they are the crowd you are being compared to. aggregate = { **ENTRY_JSON, "user": None, diff --git a/forecasting_tools/bot_review/report_parsing.py b/forecasting_tools/bot_review/report_parsing.py index 851d7b60..2de6821c 100644 --- a/forecasting_tools/bot_review/report_parsing.py +++ b/forecasting_tools/bot_review/report_parsing.py @@ -1,15 +1,11 @@ """Pull structure back out of a report explanation. -``ForecastBot._create_comment`` assembles every report into the same shape: a -SUMMARY listing each forecaster's prediction, the RESEARCH behind them, and one -FORECASTS subsection per forecaster. That string is what gets saved to disk and -what gets posted to Metaculus as a comment, so the same functions read either. +Reads the format ``ForecastBot._create_comment`` produces: a SUMMARY listing +each forecaster's prediction, the RESEARCH behind them, and one FORECASTS +subsection per forecaster. Works on a saved report or on the comment text +posted to Metaculus, which are the same string. -Sections come from ``MarkdownTree``, the same splitter ``ForecastReport`` uses. -The summary bullets are not headings, so they are read directly. - -Forecasters are keyed ``R:F``, since a bot running -more than one research report per question has a forecaster 1 in each. +Forecasters are keyed ``R:F``. """ from __future__ import annotations @@ -72,8 +68,7 @@ def forecasts_block(summary: str) -> str: def parse_forecasters(summary: str) -> list[dict]: """Each forecaster's prediction as it was written in the summary. - ``model`` is None unless the bot annotates its own bullets with model names, - which the framework does not do. + ``model`` is None unless the bot annotates its summary bullets with model names. """ reports = list(REPORT_PATTERN.finditer(summary)) blocks = _bodies_between(summary, [(m.start(), m.end()) for m in reports]) diff --git a/forecasting_tools/data_models/comment.py b/forecasting_tools/data_models/comment.py index 08289830..03aa5c93 100644 --- a/forecasting_tools/data_models/comment.py +++ b/forecasting_tools/data_models/comment.py @@ -6,16 +6,7 @@ class Comment(BaseModel): - """A comment on a Metaculus post. - - A bot publishing a report posts the report's explanation as the comment - text, so ``text`` parses with the same functions that read a saved - ``ForecastReport``. - - ``on_post`` is a post id, not a question id. The two are separate sequences - that overlap, so joining question-level data on ``on_post`` silently matches - the wrong question rather than matching nothing. - """ + """A comment on a Metaculus post. ``on_post`` is a post id, not a question id.""" id: int on_post: int diff --git a/forecasting_tools/data_models/leaderboard.py b/forecasting_tools/data_models/leaderboard.py index 59ec06c4..07c7bf64 100644 --- a/forecasting_tools/data_models/leaderboard.py +++ b/forecasting_tools/data_models/leaderboard.py @@ -6,9 +6,8 @@ class LeaderboardEntry(BaseModel): """One forecaster's standing on a leaderboard. - Community aggregates are ranked alongside forecasters. Those entries have no - user and carry an ``aggregation_method`` instead, so comparing your rank - against them is how you tell whether you beat the crowd. + Community aggregates appear as entries with no user and an + ``aggregation_method`` set. """ user_id: int | None @@ -40,11 +39,7 @@ def from_metaculus_api_json(cls, entry_json: dict) -> LeaderboardEntry: class Leaderboard(BaseModel): - """A tournament's leaderboard, plus your own entry on it. - - ``user_entry`` is None when the requesting user has no standing on this - leaderboard, which is also the case for a tournament they never forecast. - """ + """A project's leaderboard. ``user_entry`` is None if you have no standing on it.""" project_id: int project_name: str | None diff --git a/forecasting_tools/helpers/metaculus_client.py b/forecasting_tools/helpers/metaculus_client.py index a888ffcf..7319ce4a 100644 --- a/forecasting_tools/helpers/metaculus_client.py +++ b/forecasting_tools/helpers/metaculus_client.py @@ -500,18 +500,12 @@ def get_own_comments( max_comments: int = 500, ) -> list[Comment]: """ - Comments you posted, newest first. + Comments you posted, newest first. The API only allows listing your own. - The API refuses to list any other author's comments, so this is - deliberately limited to your own. - - ``is_private`` defaults to True because ``post_question_comment`` posts - private comments by default, and the endpoint omits private comments - unless asked for them. A bot's own reports are therefore invisible - without this, and an empty result is indistinguishable from one where - the bot never commented. - - Pass ``user_id`` to skip the lookup request when calling this in a loop. + :param post_id: only return comments on this post + :param is_private: whether to return private or public comments + :param user_id: your own user id, looked up if not given + :param max_comments: stop paging once this many are collected """ author_id = user_id if user_id is not None else self.get_current_user_id() comments: list[Comment] = [] @@ -550,11 +544,9 @@ def _get_comment_page(self, params: dict[str, Any]) -> list[dict]: @retry_with_exponential_backoff() def get_project_leaderboard(self, project_id: int) -> Leaderboard: """ - The primary leaderboard for a tournament, including your own entry. + The primary leaderboard for a project, including your own entry. - Takes the numeric project id; the slug that works elsewhere 404s here. - The endpoint returns every leaderboard attached to the project, of which - the primary one is the tournament's actual standings. + :param project_id: numeric project id (slugs are not accepted here) """ self._sleep_between_requests() response = requests.get( From f427c8e3a206fabed0724417a73a94713f217b21 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:26:25 +0100 Subject: [PATCH 05/21] Port the outcome table onto MetaculusClient One row per question in a tournament: the bot's latest forecast reduced to something readable, how the question resolved, and the official scores. Replaces the standalone urllib client the prototype used. The list endpoint omits my_forecasts, so scores and the bot's own forecast only come from per-post detail fetches; the list sweep is just there to enumerate the tournament and to catch questions the bot never forecast. Verified against MiniBench 2026-06-29: 60 questions, 37 forecast, 36 scored, 1 annulled, rank 17 at 348.5 -- matching the public leaderboard and the prototype's output. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_outcomes.py | 164 +++++++++++++ forecasting_tools/bot_review/outcomes.py | 232 ++++++++++++++++++ 2 files changed, 396 insertions(+) create mode 100644 code_tests/unit_tests/test_bot_review/test_outcomes.py create mode 100644 forecasting_tools/bot_review/outcomes.py diff --git a/code_tests/unit_tests/test_bot_review/test_outcomes.py b/code_tests/unit_tests/test_bot_review/test_outcomes.py new file mode 100644 index 00000000..750785cf --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_outcomes.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import math + +from forecasting_tools.bot_review.outcomes import ( + cdf_median, + outcomes_from_post, + scale_internal, + summarize_forecast, +) + + +def make_binary_post(**question_overrides) -> dict: + question = { + "id": 101, + "type": "binary", + "title": "Will X happen?", + "status": "resolved", + "resolution": "yes", + "actual_resolve_time": "2026-07-01T00:00:00Z", + "spot_scoring_time": "2026-06-15T00:00:00Z", + "my_forecasts": { + "latest": { + "forecast_values": [0.3, 0.7], + "start_time": "2026-06-10T00:00:00Z", + }, + "score_data": { + "peer_score": 5.0, + "baseline_score": 10.0, + "coverage": 0.9, + "spot_peer_score": -2.5, + }, + }, + } + question.update(question_overrides) + return { + "id": 1, + "title": "Will X happen?", + "question": question, + "projects": { + "default_project": {"id": 33064, "slug": "a-tournament", "name": "A"} + }, + } + + +class TestScaling: + def test_linear(self): + scaling = {"range_min": 0, "range_max": 100, "zero_point": None} + assert scale_internal(0.5, scaling) == 50 + + def test_log(self): + # range 1..100 with zero_point 0 -> ratio 100, so the midpoint is 10 + scaling = {"range_min": 1, "range_max": 100, "zero_point": 0} + assert math.isclose(scale_internal(0.5, scaling), 10.0) + + def test_missing_range(self): + assert scale_internal(0.5, {}) is None + + def test_cdf_median_interpolates(self): + assert math.isclose(cdf_median([0.0, 0.25, 0.75, 1.0]), 0.5) + + def test_cdf_median_endpoints(self): + assert cdf_median([0.6, 0.8, 1.0]) == 0.0 + assert cdf_median([0.0, 0.1, 0.2]) == 1.0 + + +class TestSummarizeForecast: + def test_binary(self): + assert summarize_forecast("binary", [0.3, 0.7], None, {}) == { + "probability_yes": 0.7 + } + + def test_multiple_choice_zips_options(self): + summary = summarize_forecast("multiple_choice", [0.2, 0.8], ["A", "B"], {}) + assert summary == {"probabilities": {"A": 0.2, "B": 0.8}} + + def test_numeric_median_is_scaled(self): + scaling = {"range_min": 0, "range_max": 100, "zero_point": None} + summary = summarize_forecast("numeric", [0.0, 0.25, 0.75, 1.0], None, scaling) + assert summary is not None + assert math.isclose(summary["median"], 50.0) + + def test_date_gets_an_iso_median(self): + scaling = {"range_min": 0, "range_max": 86400, "zero_point": None} + summary = summarize_forecast("date", [0.0, 0.5, 1.0], None, scaling) + assert summary is not None + assert summary["median_iso"].startswith("1970-01-01T12:00") + + def test_no_forecast(self): + assert summarize_forecast("binary", None, None, {}) is None + + +class TestOutcomesFromPost: + def test_binary_post(self): + (outcome,) = outcomes_from_post(make_binary_post()) + assert outcome.post_id == 1 + assert outcome.question_id == 101 + assert outcome.forecasted + assert outcome.forecast == {"probability_yes": 0.7} + assert outcome.resolution == "yes" + assert outcome.scores is not None + assert outcome.scores.peer_score == 5.0 + assert outcome.scores.spot_peer_score == -2.5 + assert outcome.project_slug == "a-tournament" + assert outcome.url.endswith("/questions/1/") + + def test_resolved_question_with_null_resolution(self): + post = make_binary_post( + type="multiple_choice", + resolution=None, + options=["0", "1"], + my_forecasts={"latest": {"forecast_values": [0.4, 0.6]}, "score_data": {}}, + ) + (outcome,) = outcomes_from_post(post) + assert outcome.status == "resolved" + assert outcome.resolution is None + assert outcome.forecast == {"probabilities": {"0": 0.4, "1": 0.6}} + assert outcome.scores is None + + def test_annulled_question(self): + post = make_binary_post(resolution="annulled", my_forecasts={}) + (outcome,) = outcomes_from_post(post) + assert outcome.was_annulled + assert not outcome.forecasted + + def test_unforecasted_question(self): + (outcome,) = outcomes_from_post(make_binary_post(my_forecasts={})) + assert not outcome.forecasted + assert outcome.forecast is None + assert outcome.scores is None + + def test_group_post_expands_to_one_row_per_subquestion(self): + subquestion = make_binary_post()["question"] + post = { + "id": 2, + "title": "Group", + "projects": {}, + "group_of_questions": { + "questions": [ + dict(subquestion, id=201, label="a"), + dict(subquestion, id=202, label="b"), + ] + }, + } + outcomes = outcomes_from_post(post) + assert [o.question_id for o in outcomes] == [201, 202] + assert [o.group_label for o in outcomes] == ["a", "b"] + assert all(o.post_id == 2 for o in outcomes) + + def test_conditional_post_expands_to_both_branches(self): + child = make_binary_post()["question"] + post = { + "id": 3, + "projects": {}, + "conditional": { + "question_yes": dict(child, id=301), + "question_no": dict(child, id=302), + }, + } + outcomes = outcomes_from_post(post) + assert [o.conditional_branch for o in outcomes] == ["yes", "no"] + + def test_notebooks_are_skipped(self): + assert outcomes_from_post({"id": 4, "notebook": {}}) == [] diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py new file mode 100644 index 00000000..d0518c8d --- /dev/null +++ b/forecasting_tools/bot_review/outcomes.py @@ -0,0 +1,232 @@ +"""Build a table of how a bot's forecasts turned out. + +One row per question in a tournament: the bot's latest forecast, how the +question resolved, and the scores Metaculus computed for it. +""" + +from __future__ import annotations + +import asyncio +import logging +from datetime import datetime, timezone +from typing import Any + +from pydantic import BaseModel + +from forecasting_tools.data_models.leaderboard import LeaderboardEntry +from forecasting_tools.helpers.metaculus_client import ApiFilter, MetaculusClient + +logger = logging.getLogger(__name__) + +CONTINUOUS_TYPES = ("numeric", "date", "discrete") +MAX_QUESTIONS_PER_TOURNAMENT = 1000 + + +class QuestionScores(BaseModel): + """Official scores from ``my_forecasts.score_data``.""" + + peer_score: float | None = None + baseline_score: float | None = None + spot_peer_score: float | None = None + spot_baseline_score: float | None = None + coverage: float | None = None + weighted_coverage: float | None = None + relative_legacy_score: float | None = None + + +class QuestionOutcome(BaseModel): + """A question, the bot's forecast on it, and the result.""" + + post_id: int + question_id: int + title: str | None + url: str + question_type: str | None + status: str | None + resolution: str | None + actual_resolve_time: datetime | None + spot_scoring_time: datetime | None + options: list[str] | None + group_label: str | None = None + conditional_branch: str | None = None + forecasted: bool + forecast: dict[str, Any] | None + forecast_time: datetime | None + scores: QuestionScores | None + project_id: int | None + project_slug: str | None + project_name: str | None + + @property + def was_annulled(self) -> bool: + return self.resolution == "annulled" + + +class TournamentOutcomes(BaseModel): + """Every question in a tournament, with the bot's standing on it.""" + + generated_at: datetime + project_id: int | None + project_slug: str | None + project_name: str | None + user_id: int + leaderboard_entry: LeaderboardEntry | None + questions: list[QuestionOutcome] + + +def scale_internal(location: float, scaling: dict) -> float | None: + """Map an internal [0, 1] location onto the question's own units.""" + range_min, range_max = scaling.get("range_min"), scaling.get("range_max") + zero_point = scaling.get("zero_point") + if range_min is None or range_max is None: + return None + if zero_point is not None: + ratio = (range_max - zero_point) / (range_min - zero_point) + return range_min + (range_max - range_min) * (ratio**location - 1) / (ratio - 1) + return range_min + location * (range_max - range_min) + + +def cdf_median(cdf: list[float]) -> float: + """The internal [0, 1] location where the CDF crosses 0.5.""" + for index, value in enumerate(cdf): + if value >= 0.5: + if index == 0: + return 0.0 + below = cdf[index - 1] + fraction = 0.0 if value == below else (0.5 - below) / (value - below) + return (index - 1 + fraction) / (len(cdf) - 1) + return 1.0 + + +def summarize_forecast( + question_type: str | None, + forecast_values: list[float] | None, + options: list[str] | None, + scaling: dict, +) -> dict[str, Any] | None: + """Reduce raw ``forecast_values`` to a readable summary of the forecast.""" + if not forecast_values: + return None + if question_type == "binary": + probability = ( + forecast_values[-1] if len(forecast_values) == 2 else forecast_values[0] + ) + return {"probability_yes": probability} + if question_type == "multiple_choice": + return {"probabilities": dict(zip(options or [], forecast_values))} + if question_type in CONTINUOUS_TYPES: + median = scale_internal(cdf_median(forecast_values), scaling) + summary: dict[str, Any] = {"median": median} + if question_type == "date" and median is not None: + summary["median_iso"] = datetime.fromtimestamp( + median, tz=timezone.utc + ).isoformat() + return summary + return {"values": forecast_values} + + +def _question_outcome( + post_json: dict, question_json: dict, **extra: Any +) -> QuestionOutcome: + post_id = post_json["id"] + question_type = question_json.get("type") + options = question_json.get("options") + scaling = question_json.get("scaling") or {} + project = (post_json.get("projects") or {}).get("default_project") or {} + my_forecasts = question_json.get("my_forecasts") or {} + latest = my_forecasts.get("latest") or {} + score_data = my_forecasts.get("score_data") or None + return QuestionOutcome( + post_id=post_id, + question_id=question_json["id"], + title=question_json.get("title") or post_json.get("title"), + url=f"https://www.metaculus.com/questions/{post_id}/", + question_type=question_type, + status=question_json.get("status") or post_json.get("status"), + resolution=question_json.get("resolution"), + actual_resolve_time=question_json.get("actual_resolve_time"), + spot_scoring_time=question_json.get("spot_scoring_time"), + options=options, + forecasted=bool(latest), + forecast=summarize_forecast( + question_type, latest.get("forecast_values"), options, scaling + ), + forecast_time=latest.get("start_time"), + scores=QuestionScores(**score_data) if score_data else None, + project_id=project.get("id"), + project_slug=project.get("slug"), + project_name=project.get("name"), + **extra, + ) + + +def outcomes_from_post(post_json: dict) -> list[QuestionOutcome]: + """One outcome per question on a post, unpacking groups and conditionals.""" + if post_json.get("notebook"): + return [] + if post_json.get("question"): + return [_question_outcome(post_json, post_json["question"])] + group = post_json.get("group_of_questions") + if group: + return [ + _question_outcome(post_json, question, group_label=question.get("label")) + for question in group.get("questions", []) + ] + conditional = post_json.get("conditional") + if conditional: + return [ + _question_outcome( + post_json, conditional[branch], conditional_branch=branch.split("_")[1] + ) + for branch in ("question_yes", "question_no") + if conditional.get(branch) + ] + return [] + + +def get_tournament_outcomes( + tournament: int | str, client: MetaculusClient | None = None +) -> TournamentOutcomes: + """ + Fetch every question in a tournament and how the bot did on each. + + :param tournament: tournament slug or id + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + api_filter = ApiFilter( + allowed_tournaments=[tournament], + group_question_mode="unpack_subquestions", + ) + listed_questions = asyncio.run( + client.get_questions_matching_filter( + api_filter, + num_questions=MAX_QUESTIONS_PER_TOURNAMENT, + error_if_question_target_missed=False, + ) + ) + post_ids = list(dict.fromkeys(question.id_of_post for question in listed_questions)) + logger.info(f"Building outcomes for {len(post_ids)} posts in {tournament}") + + outcomes: list[QuestionOutcome] = [] + for post_id in post_ids: + assert post_id is not None + fetched = client.get_question_by_post_id( + post_id, group_question_mode="unpack_subquestions" + ) + questions = fetched if isinstance(fetched, list) else [fetched] + outcomes.extend(outcomes_from_post(questions[0].api_json)) + + project_id = outcomes[0].project_id if outcomes else None + leaderboard = ( + client.get_project_leaderboard(project_id) if project_id is not None else None + ) + return TournamentOutcomes( + generated_at=datetime.now(tz=timezone.utc), + project_id=project_id, + project_slug=outcomes[0].project_slug if outcomes else None, + project_name=outcomes[0].project_name if outcomes else None, + user_id=client.get_current_user_id(), + leaderboard_entry=leaderboard.user_entry if leaderboard else None, + questions=outcomes, + ) From 5e55ac34c42fc9a52441bdf6925b12067f40ffd7 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:42:22 +0100 Subject: [PATCH 06/21] Let the outcome table skip questions the bot never forecast Adds forecasted_only, which filters server-side with forecaster_id, and get_outcomes_for_posts for reviewing a handful of questions directly. Co-Authored-By: Claude Opus 5 --- forecasting_tools/bot_review/outcomes.py | 38 +++++++++++++++++------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py index d0518c8d..5af2db2a 100644 --- a/forecasting_tools/bot_review/outcomes.py +++ b/forecasting_tools/bot_review/outcomes.py @@ -184,19 +184,43 @@ def outcomes_from_post(post_json: dict) -> list[QuestionOutcome]: return [] +def get_outcomes_for_posts( + post_ids: list[int], client: MetaculusClient | None = None +) -> list[QuestionOutcome]: + """ + Fetch outcomes for specific posts, one request each. + + :param post_ids: post ids, as they appear in question urls + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + outcomes: list[QuestionOutcome] = [] + for post_id in post_ids: + fetched = client.get_question_by_post_id( + post_id, group_question_mode="unpack_subquestions" + ) + questions = fetched if isinstance(fetched, list) else [fetched] + outcomes.extend(outcomes_from_post(questions[0].api_json)) + return outcomes + + def get_tournament_outcomes( - tournament: int | str, client: MetaculusClient | None = None + tournament: int | str, + client: MetaculusClient | None = None, + forecasted_only: bool = False, ) -> TournamentOutcomes: """ - Fetch every question in a tournament and how the bot did on each. + Fetch the questions in a tournament and how the bot did on each. :param tournament: tournament slug or id :param client: client to fetch with, created from the environment if not given + :param forecasted_only: skip questions the bot never forecast """ client = client or MetaculusClient() api_filter = ApiFilter( allowed_tournaments=[tournament], group_question_mode="unpack_subquestions", + is_previously_forecasted_by_user=True if forecasted_only else None, ) listed_questions = asyncio.run( client.get_questions_matching_filter( @@ -208,15 +232,7 @@ def get_tournament_outcomes( post_ids = list(dict.fromkeys(question.id_of_post for question in listed_questions)) logger.info(f"Building outcomes for {len(post_ids)} posts in {tournament}") - outcomes: list[QuestionOutcome] = [] - for post_id in post_ids: - assert post_id is not None - fetched = client.get_question_by_post_id( - post_id, group_question_mode="unpack_subquestions" - ) - questions = fetched if isinstance(fetched, list) else [fetched] - outcomes.extend(outcomes_from_post(questions[0].api_json)) - + outcomes = get_outcomes_for_posts(post_ids, client) # type: ignore[arg-type] project_id = outcomes[0].project_id if outcomes else None leaderboard = ( client.get_project_leaderboard(project_id) if project_id is not None else None From b9c244196edb5aed8519ea2aa1924bfcfe4e4999 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:46:44 +0100 Subject: [PATCH 07/21] Add a bot-review CLI instead of a per-repo script Registered as a poetry script alongside source-archive, so a bot maker gets it from installing forecasting-tools rather than copying a driver into their own repo. Writes the table as json for later stages to read. Co-Authored-By: Claude Opus 5 --- forecasting_tools/bot_review/cli.py | 79 +++++++++++++++++++++++++++++ pyproject.toml | 1 + 2 files changed, 80 insertions(+) create mode 100644 forecasting_tools/bot_review/cli.py diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py new file mode 100644 index 00000000..4af2be4a --- /dev/null +++ b/forecasting_tools/bot_review/cli.py @@ -0,0 +1,79 @@ +"""Command line entry point for reviewing a bot's forecasts.""" + +from __future__ import annotations + +import argparse +import json +import logging + +from forecasting_tools.bot_review.outcomes import ( + QuestionOutcome, + get_outcomes_for_posts, + get_tournament_outcomes, +) +from forecasting_tools.helpers.metaculus_client import MetaculusClient + + +def _print_outcome(outcome: QuestionOutcome) -> None: + print(f"{outcome.post_id} {outcome.question_type or '':16} {outcome.title or ''}") + print(f" status={outcome.status} resolution={outcome.resolution}") + print(f" forecast={outcome.forecast}") + print(f" scores={outcome.scores}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Review how a bot's forecasts did") + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument("--tournament", help="tournament slug or id") + source.add_argument("--post", type=int, nargs="+", metavar="POST_ID") + parser.add_argument( + "--all", + action="store_true", + help="include tournament questions the bot never forecast", + ) + parser.add_argument("--output", help="write the full table to this json file") + parser.add_argument( + "--seconds-between-requests", + type=float, + default=0.7, + help="delay between Metaculus requests", + ) + args = parser.parse_args() + logging.basicConfig(level=logging.WARNING) + + client = MetaculusClient( + sleep_seconds_between_requests=args.seconds_between_requests + ) + if args.post: + outcomes = get_outcomes_for_posts(args.post, client) + report = None + else: + report = get_tournament_outcomes( + args.tournament, client=client, forecasted_only=not args.all + ) + outcomes = report.questions + print(f"\n{report.project_name} ({report.project_id}) as user {report.user_id}") + print(f"leaderboard: {report.leaderboard_entry}") + + scored = [outcome for outcome in outcomes if outcome.scores] + forecasted = [outcome for outcome in outcomes if outcome.forecasted] + print( + f"questions {len(outcomes)} | forecasted {len(forecasted)} | scored {len(scored)}\n" + ) + for outcome in outcomes: + _print_outcome(outcome) + + if args.output: + if report is not None: + json_text = report.model_dump_json(indent=2) + else: + json_text = json.dumps( + [outcome.model_dump(mode="json") for outcome in outcomes], indent=2 + ) + with open(args.output, "w") as file: + file.write(json_text) + print(f"\nwrote {args.output}") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index d15ad580..da2c7ee5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ source-archive = ["boto3", "playwright", "firecrawl-py", "trafilatura"] [tool.poetry.scripts] source-archive = "forecasting_tools.agents_and_tools.source_archive.cli:main" +bot-review = "forecasting_tools.bot_review.cli:main" [tool.poetry.group.dev.dependencies] time-machine = ">=2.19.0,<4.0.0" From ef8a5701535fc1a6bc765f68f43fac9caf767670 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:53:21 +0100 Subject: [PATCH 08/21] Report which user the bot review is acting as Loads a local .env like the source-archive CLI does, and prints the user id in every mode, since my_forecasts is scoped to the token and an ambient one otherwise makes the review silently report no forecasts. Co-Authored-By: Claude Opus 5 --- forecasting_tools/bot_review/cli.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index 4af2be4a..41e05331 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -6,6 +6,8 @@ import json import logging +from dotenv import load_dotenv + from forecasting_tools.bot_review.outcomes import ( QuestionOutcome, get_outcomes_for_posts, @@ -40,10 +42,12 @@ def main() -> None: ) args = parser.parse_args() logging.basicConfig(level=logging.WARNING) + load_dotenv() client = MetaculusClient( sleep_seconds_between_requests=args.seconds_between_requests ) + print(f"\nreviewing forecasts made by user {client.get_current_user_id()}") if args.post: outcomes = get_outcomes_for_posts(args.post, client) report = None @@ -52,7 +56,7 @@ def main() -> None: args.tournament, client=client, forecasted_only=not args.all ) outcomes = report.questions - print(f"\n{report.project_name} ({report.project_id}) as user {report.user_id}") + print(f"{report.project_name} ({report.project_id})") print(f"leaderboard: {report.leaderboard_entry}") scored = [outcome for outcome in outcomes if outcome.scores] From e14719be2c18070df0e35aa7b69637388b068256 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 22:55:47 +0100 Subject: [PATCH 09/21] Rename --all to --include-unforecasted --all read as every question on the site rather than every question in the tournament. Co-Authored-By: Claude Opus 5 --- forecasting_tools/bot_review/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index 41e05331..f7cbe932 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -29,9 +29,9 @@ def main() -> None: source.add_argument("--tournament", help="tournament slug or id") source.add_argument("--post", type=int, nargs="+", metavar="POST_ID") parser.add_argument( - "--all", + "--include-unforecasted", action="store_true", - help="include tournament questions the bot never forecast", + help="also include questions in the tournament that the bot never forecast", ) parser.add_argument("--output", help="write the full table to this json file") parser.add_argument( @@ -53,7 +53,9 @@ def main() -> None: report = None else: report = get_tournament_outcomes( - args.tournament, client=client, forecasted_only=not args.all + args.tournament, + client=client, + forecasted_only=not args.include_unforecasted, ) outcomes = report.questions print(f"{report.project_name} ({report.project_id})") From ed34ee51973d05fa88d2d53f1defe10b05ff3164 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:00:59 +0100 Subject: [PATCH 10/21] Add --resolved-since for questions that resolved recently Spans tournaments, so it is the query a scheduled review runs rather than one tied to a competition. Narrows server-side on scheduled resolve time, then filters on the actual one. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_outcomes.py | 38 +++++++++++++++++++ forecasting_tools/bot_review/cli.py | 10 +++++ forecasting_tools/bot_review/outcomes.py | 37 +++++++++++++++++- 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/code_tests/unit_tests/test_bot_review/test_outcomes.py b/code_tests/unit_tests/test_bot_review/test_outcomes.py index 750785cf..1c40975d 100644 --- a/code_tests/unit_tests/test_bot_review/test_outcomes.py +++ b/code_tests/unit_tests/test_bot_review/test_outcomes.py @@ -1,9 +1,13 @@ from __future__ import annotations import math +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, MagicMock, patch +from forecasting_tools.bot_review import outcomes as outcomes_module from forecasting_tools.bot_review.outcomes import ( cdf_median, + get_recently_resolved_outcomes, outcomes_from_post, scale_internal, summarize_forecast, @@ -162,3 +166,37 @@ def test_conditional_post_expands_to_both_branches(self): def test_notebooks_are_skipped(self): assert outcomes_from_post({"id": 4, "notebook": {}}) == [] + + +class TestRecentlyResolved: + def outcome_resolved(self, resolve_time: datetime | None): + (outcome,) = outcomes_from_post(make_binary_post()) + return outcome.model_copy(update={"actual_resolve_time": resolve_time}) + + def test_keeps_only_questions_resolved_inside_the_window(self): + now = datetime.now(tz=timezone.utc) + recent = self.outcome_resolved(now - timedelta(days=2)) + old = self.outcome_resolved(now - timedelta(days=40)) + never = self.outcome_resolved(None) + + client = MagicMock() + client.get_questions_matching_filter = AsyncMock(return_value=[]) + with patch.object( + outcomes_module, + "get_outcomes_for_posts", + return_value=[recent, old, never], + ): + kept = get_recently_resolved_outcomes(7, client) + + assert kept == [recent] + + def test_asks_the_api_for_resolved_questions_it_forecast(self): + client = MagicMock() + client.get_questions_matching_filter = AsyncMock(return_value=[]) + with patch.object(outcomes_module, "get_outcomes_for_posts", return_value=[]): + get_recently_resolved_outcomes(7, client) + + api_filter = client.get_questions_matching_filter.call_args.args[0] + assert api_filter.allowed_statuses == ["resolved"] + assert api_filter.is_previously_forecasted_by_user + assert api_filter.scheduled_resolve_time_gt is not None diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index f7cbe932..ee279400 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -11,6 +11,7 @@ from forecasting_tools.bot_review.outcomes import ( QuestionOutcome, get_outcomes_for_posts, + get_recently_resolved_outcomes, get_tournament_outcomes, ) from forecasting_tools.helpers.metaculus_client import MetaculusClient @@ -28,6 +29,12 @@ def main() -> None: source = parser.add_mutually_exclusive_group(required=True) source.add_argument("--tournament", help="tournament slug or id") source.add_argument("--post", type=int, nargs="+", metavar="POST_ID") + source.add_argument( + "--resolved-since", + type=int, + metavar="DAYS", + help="your questions that resolved in the last DAYS days, any tournament", + ) parser.add_argument( "--include-unforecasted", action="store_true", @@ -51,6 +58,9 @@ def main() -> None: if args.post: outcomes = get_outcomes_for_posts(args.post, client) report = None + elif args.resolved_since: + outcomes = get_recently_resolved_outcomes(args.resolved_since, client) + report = None else: report = get_tournament_outcomes( args.tournament, diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py index 5af2db2a..187345e4 100644 --- a/forecasting_tools/bot_review/outcomes.py +++ b/forecasting_tools/bot_review/outcomes.py @@ -8,7 +8,7 @@ import asyncio import logging -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from typing import Any from pydantic import BaseModel @@ -204,6 +204,41 @@ def get_outcomes_for_posts( return outcomes +def get_recently_resolved_outcomes( + days: int, client: MetaculusClient | None = None +) -> list[QuestionOutcome]: + """ + Fetch the bot's forecasts on questions that resolved in the last few days. + + :param days: how far back to look + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + cutoff = datetime.now(tz=timezone.utc) - timedelta(days=days) + api_filter = ApiFilter( + allowed_statuses=["resolved"], + is_previously_forecasted_by_user=True, + scheduled_resolve_time_gt=cutoff, + group_question_mode="unpack_subquestions", + ) + listed_questions = asyncio.run( + client.get_questions_matching_filter( + api_filter, + num_questions=MAX_QUESTIONS_PER_TOURNAMENT, + error_if_question_target_missed=False, + ) + ) + post_ids = list(dict.fromkeys(question.id_of_post for question in listed_questions)) + logger.info(f"Building outcomes for {len(post_ids)} posts resolved since {cutoff}") + outcomes = get_outcomes_for_posts(post_ids, client) # type: ignore[arg-type] + return [ + outcome + for outcome in outcomes + if outcome.actual_resolve_time is not None + and outcome.actual_resolve_time >= cutoff + ] + + def get_tournament_outcomes( tournament: int | str, client: MetaculusClient | None = None, From 1f6b0f5e0bdfc4ae68bb694faf92e43d656fb8bf Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:05:14 +0100 Subject: [PATCH 11/21] Widen the resolved-since API window for late resolutions The API filters on scheduled resolve time, so a question that resolved later than scheduled can fall outside the server-side window while sitting inside the requested one. Measured at 1 of 44 resolved questions, 2.6 days late; the window now reaches 14 days further back and the local filter on actual resolve time stays exact. Co-Authored-By: Claude Opus 5 --- .../unit_tests/test_bot_review/test_outcomes.py | 11 +++++++++++ forecasting_tools/bot_review/outcomes.py | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/code_tests/unit_tests/test_bot_review/test_outcomes.py b/code_tests/unit_tests/test_bot_review/test_outcomes.py index 1c40975d..464f88f4 100644 --- a/code_tests/unit_tests/test_bot_review/test_outcomes.py +++ b/code_tests/unit_tests/test_bot_review/test_outcomes.py @@ -200,3 +200,14 @@ def test_asks_the_api_for_resolved_questions_it_forecast(self): assert api_filter.allowed_statuses == ["resolved"] assert api_filter.is_previously_forecasted_by_user assert api_filter.scheduled_resolve_time_gt is not None + + def test_the_api_window_reaches_further_back_than_the_cutoff(self): + # a question scheduled before the cutoff can still resolve inside it + client = MagicMock() + client.get_questions_matching_filter = AsyncMock(return_value=[]) + with patch.object(outcomes_module, "get_outcomes_for_posts", return_value=[]): + get_recently_resolved_outcomes(7, client) + + api_filter = client.get_questions_matching_filter.call_args.args[0] + cutoff = datetime.now(tz=timezone.utc) - timedelta(days=7) + assert api_filter.scheduled_resolve_time_gt < cutoff diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py index 187345e4..793964d0 100644 --- a/forecasting_tools/bot_review/outcomes.py +++ b/forecasting_tools/bot_review/outcomes.py @@ -20,6 +20,7 @@ CONTINUOUS_TYPES = ("numeric", "date", "discrete") MAX_QUESTIONS_PER_TOURNAMENT = 1000 +LATE_RESOLUTION_ALLOWANCE = timedelta(days=14) class QuestionScores(BaseModel): @@ -218,7 +219,7 @@ def get_recently_resolved_outcomes( api_filter = ApiFilter( allowed_statuses=["resolved"], is_previously_forecasted_by_user=True, - scheduled_resolve_time_gt=cutoff, + scheduled_resolve_time_gt=cutoff - LATE_RESOLUTION_ALLOWANCE, group_question_mode="unpack_subquestions", ) listed_questions = asyncio.run( From c4715cde0735d9dddfbc48bef0b72070face34b5 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:14:44 +0100 Subject: [PATCH 12/21] Add the markdown summary report over the outcome table Renames TournamentOutcomes to OutcomeTable and stores the whole Leaderboard rather than just our entry, so the report can print the rank denominator and score type. The CLI now prints the summary instead of dumping every row, and --summary writes it to a file. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_summary.py | 172 ++++++++++++++++++ forecasting_tools/bot_review/cli.py | 68 ++++--- forecasting_tools/bot_review/outcomes.py | 20 +- forecasting_tools/bot_review/summary.py | 124 +++++++++++++ 4 files changed, 338 insertions(+), 46 deletions(-) create mode 100644 code_tests/unit_tests/test_bot_review/test_summary.py create mode 100644 forecasting_tools/bot_review/summary.py diff --git a/code_tests/unit_tests/test_bot_review/test_summary.py b/code_tests/unit_tests/test_bot_review/test_summary.py new file mode 100644 index 00000000..bf8daab8 --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_summary.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from forecasting_tools.bot_review.outcomes import ( + OutcomeTable, + QuestionOutcome, + QuestionScores, +) +from forecasting_tools.bot_review.summary import build_summary +from forecasting_tools.data_models.leaderboard import Leaderboard, LeaderboardEntry + + +def make_question( + post_id: int, + spot_peer: float | None = 0.0, + baseline: float | None = 0.0, + scored: bool = True, + forecasted: bool = True, + resolution: str | None = "yes", + project: str = "CupA", + status: str = "resolved", +) -> QuestionOutcome: + return QuestionOutcome( + post_id=post_id, + question_id=post_id + 100, + title=f"Q{post_id}", + url=f"https://www.metaculus.com/questions/{post_id}/", + question_type="binary", + status=status, + resolution=resolution, + actual_resolve_time=None, + spot_scoring_time=None, + options=None, + forecasted=forecasted, + forecast={"probability_yes": 0.5} if forecasted else None, + forecast_time=None, + scores=( + QuestionScores( + spot_peer_score=spot_peer, baseline_score=baseline, coverage=0.5 + ) + if scored + else None + ), + project_id=1, + project_slug=project.lower(), + project_name=project, + ) + + +def make_leaderboard(rank: int = 3, entries: int = 50) -> Leaderboard: + def entry(user_id: int | None, rank: int | None) -> LeaderboardEntry: + return LeaderboardEntry( + user_id=user_id, + username=f"bot{user_id}", + aggregation_method=None, + rank=rank, + score=30.0, + coverage=4.0, + contribution_count=4, + medal=None, + prize=None, + excluded=False, + ) + + return Leaderboard( + project_id=1, + project_name="TestCup", + project_slug="testcup", + score_type="spot_peer_tournament", + finalized=False, + entries=[entry(i, i) for i in range(1, entries + 1)], + user_entry=entry(7, rank), + ) + + +def make_table(**overrides) -> OutcomeTable: + fields = { + "generated_at": datetime(2026, 7, 23, tzinfo=timezone.utc), + "user_id": 7, + "project_name": "TestCup", + "leaderboard": make_leaderboard(), + "questions": [ + make_question(1, 20.0, 40.0), + make_question(2, -15.0, -30.0), + make_question(3, 25.0, 50.0), + make_question(4, 0.0, 5.0), + make_question(5, scored=False, resolution="annulled"), + make_question(6, scored=False, forecasted=False, resolution=None), + ], + } + fields.update(overrides) + return OutcomeTable(**fields) + + +def test_counts(): + report = build_summary(make_table()) + assert "Questions: 6" in report + assert "Forecasted: 5" in report + assert "Scored: 4" in report + assert "Forecasted but unscored: 1 (1 resolved without a score)" in report + + +def test_single_project_still_shows_breakdown(): + report = build_summary(make_table()) + assert "### By tournament" in report + assert "- CupA: 6 questions (4 scored)" in report + + +def test_multi_project_breakdown_counts_questions_and_scored(): + questions = make_table().questions + for index in (0, 4): + questions[index] = questions[index].model_copy( + update={"project_name": "CupB", "project_slug": "cupb"} + ) + report = build_summary(make_table(questions=questions)) + assert "- CupA: 4 questions (3 scored)" in report + assert "- CupB: 2 questions (1 scored)" in report + + +def test_annulled_listed_with_resolution(): + report = build_summary(make_table()) + assert "Q5" in report and "resolution: annulled" in report + + +def test_best_worst_ordering(): + report = build_summary(make_table(), top_n=2) + best = report.split("## Best")[1].split("## Worst")[0] + worst = report.split("## Worst")[1] + assert best.index("Q3") < best.index("Q1") + assert worst.index("Q2") < worst.index("Q4") + assert "Q2" not in best + assert "Q3" not in worst + + +def test_signed_formatting_and_leaderboard(): + report = build_summary(make_table()) + assert "+25.0 spot-peer" in report + assert "-15.0 spot-peer" in report + assert "Rank: **3 / 50** (spot_peer_tournament)" in report + + +def test_mean_coverage_over_scored(): + report = build_summary(make_table()) + assert "Mean coverage: 0.50 (over 4 scored questions)" in report + + +def test_open_unscored_question_not_listed_as_resolved(): + questions = make_table().questions + questions[4] = questions[4].model_copy(update={"status": "open"}) + report = build_summary(make_table(questions=questions)) + assert "Forecasted but unscored: 1 (0 resolved without a score)" in report + assert "Q5" not in report + + +def test_no_leaderboard_entry(): + assert "No leaderboard entry found." in build_summary(make_table(leaderboard=None)) + + +def test_questions_without_a_spot_peer_score_are_not_ranked(): + questions = [make_question(1, spot_peer=None, baseline=40.0), make_question(2, 5.0)] + report = build_summary(make_table(questions=questions)) + assert "Scored: 2" in report + assert "## Best 1 (by spot peer score)" in report + assert "Q1" not in report.split("## Best")[1] + + +def test_no_scored_questions(): + questions = [make_question(1, scored=False, status="open")] + report = build_summary(make_table(questions=questions, leaderboard=None)) + assert "## Best 0 (by spot peer score)" in report + assert "Mean coverage" not in report diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index ee279400..76983cc1 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -3,27 +3,21 @@ from __future__ import annotations import argparse -import json import logging +from datetime import datetime, timezone from dotenv import load_dotenv from forecasting_tools.bot_review.outcomes import ( - QuestionOutcome, + OutcomeTable, get_outcomes_for_posts, get_recently_resolved_outcomes, get_tournament_outcomes, ) +from forecasting_tools.bot_review.summary import build_summary from forecasting_tools.helpers.metaculus_client import MetaculusClient -def _print_outcome(outcome: QuestionOutcome) -> None: - print(f"{outcome.post_id} {outcome.question_type or '':16} {outcome.title or ''}") - print(f" status={outcome.status} resolution={outcome.resolution}") - print(f" forecast={outcome.forecast}") - print(f" scores={outcome.scores}") - - def main() -> None: parser = argparse.ArgumentParser(description="Review how a bot's forecasts did") source = parser.add_mutually_exclusive_group(required=True) @@ -41,6 +35,10 @@ def main() -> None: help="also include questions in the tournament that the bot never forecast", ) parser.add_argument("--output", help="write the full table to this json file") + parser.add_argument("--summary", help="write the markdown report to this file") + parser.add_argument( + "--top", type=int, default=10, help="how many best and worst questions to list" + ) parser.add_argument( "--seconds-between-requests", type=float, @@ -54,41 +52,39 @@ def main() -> None: client = MetaculusClient( sleep_seconds_between_requests=args.seconds_between_requests ) - print(f"\nreviewing forecasts made by user {client.get_current_user_id()}") - if args.post: - outcomes = get_outcomes_for_posts(args.post, client) - report = None - elif args.resolved_since: - outcomes = get_recently_resolved_outcomes(args.resolved_since, client) - report = None - else: - report = get_tournament_outcomes( + user_id = client.get_current_user_id() + print(f"\nreviewing forecasts made by user {user_id}") + if args.tournament: + table = get_tournament_outcomes( args.tournament, client=client, forecasted_only=not args.include_unforecasted, ) - outcomes = report.questions - print(f"{report.project_name} ({report.project_id})") - print(f"leaderboard: {report.leaderboard_entry}") + else: + if args.post: + outcomes = get_outcomes_for_posts(args.post, client) + project_name = "selected questions" + else: + outcomes = get_recently_resolved_outcomes(args.resolved_since, client) + project_name = f"questions resolved in the last {args.resolved_since} days" + table = OutcomeTable( + generated_at=datetime.now(tz=timezone.utc), + user_id=user_id, + questions=outcomes, + project_name=project_name, + ) - scored = [outcome for outcome in outcomes if outcome.scores] - forecasted = [outcome for outcome in outcomes if outcome.forecasted] - print( - f"questions {len(outcomes)} | forecasted {len(forecasted)} | scored {len(scored)}\n" - ) - for outcome in outcomes: - _print_outcome(outcome) + report = build_summary(table, top_n=args.top) + print(report) if args.output: - if report is not None: - json_text = report.model_dump_json(indent=2) - else: - json_text = json.dumps( - [outcome.model_dump(mode="json") for outcome in outcomes], indent=2 - ) with open(args.output, "w") as file: - file.write(json_text) - print(f"\nwrote {args.output}") + file.write(table.model_dump_json(indent=2)) + print(f"wrote {args.output}") + if args.summary: + with open(args.summary, "w") as file: + file.write(report) + print(f"wrote {args.summary}") if __name__ == "__main__": diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py index 793964d0..325d7f22 100644 --- a/forecasting_tools/bot_review/outcomes.py +++ b/forecasting_tools/bot_review/outcomes.py @@ -13,7 +13,7 @@ from pydantic import BaseModel -from forecasting_tools.data_models.leaderboard import LeaderboardEntry +from forecasting_tools.data_models.leaderboard import Leaderboard from forecasting_tools.helpers.metaculus_client import ApiFilter, MetaculusClient logger = logging.getLogger(__name__) @@ -63,16 +63,16 @@ def was_annulled(self) -> bool: return self.resolution == "annulled" -class TournamentOutcomes(BaseModel): - """Every question in a tournament, with the bot's standing on it.""" +class OutcomeTable(BaseModel): + """A set of questions, the bot's forecasts on them, and its standing.""" generated_at: datetime - project_id: int | None - project_slug: str | None - project_name: str | None user_id: int - leaderboard_entry: LeaderboardEntry | None questions: list[QuestionOutcome] + project_id: int | None = None + project_slug: str | None = None + project_name: str | None = None + leaderboard: Leaderboard | None = None def scale_internal(location: float, scaling: dict) -> float | None: @@ -244,7 +244,7 @@ def get_tournament_outcomes( tournament: int | str, client: MetaculusClient | None = None, forecasted_only: bool = False, -) -> TournamentOutcomes: +) -> OutcomeTable: """ Fetch the questions in a tournament and how the bot did on each. @@ -273,12 +273,12 @@ def get_tournament_outcomes( leaderboard = ( client.get_project_leaderboard(project_id) if project_id is not None else None ) - return TournamentOutcomes( + return OutcomeTable( generated_at=datetime.now(tz=timezone.utc), project_id=project_id, project_slug=outcomes[0].project_slug if outcomes else None, project_name=outcomes[0].project_name if outcomes else None, user_id=client.get_current_user_id(), - leaderboard_entry=leaderboard.user_entry if leaderboard else None, + leaderboard=leaderboard, questions=outcomes, ) diff --git a/forecasting_tools/bot_review/summary.py b/forecasting_tools/bot_review/summary.py new file mode 100644 index 00000000..fcc4d27a --- /dev/null +++ b/forecasting_tools/bot_review/summary.py @@ -0,0 +1,124 @@ +"""Turn an outcome table into a markdown report. + +Covers the bot's leaderboard standing, how many questions it forecast and +scored, and the questions it did best and worst on. +""" + +from __future__ import annotations + +from collections import Counter + +from forecasting_tools.bot_review.outcomes import OutcomeTable, QuestionOutcome + +RANK_METRIC = "spot_peer_score" + + +def _signed(value: float | None) -> str: + return "n/a" if value is None else f"{value:+.1f}" + + +def _project_label(outcome: QuestionOutcome) -> str: + return outcome.project_name or outcome.project_slug or str(outcome.project_id) + + +def _question_line(rank: int, outcome: QuestionOutcome) -> str: + assert outcome.scores is not None + return ( + f"{rank}. {_signed(outcome.scores.spot_peer_score)} spot-peer · " + f"{_signed(outcome.scores.baseline_score)} baseline · " + f"{outcome.question_type} · [{outcome.title}]({outcome.url})" + ) + + +def _overall_lines(table: OutcomeTable, scored: list[QuestionOutcome]) -> list[str]: + lines = ["## Overall", ""] + leaderboard = table.leaderboard + entry = leaderboard.user_entry if leaderboard else None + if leaderboard is not None and entry is not None: + lines += [ + ( + f"- Rank: **{entry.rank} / {len(leaderboard.entries)}** " + f"({leaderboard.score_type})" + ), + ( + f"- Score: **{entry.score:.2f}**" + if entry.score is not None + else "- Score: n/a" + ), + f"- Medal: {entry.medal or 'none'}", + ] + else: + lines.append("- No leaderboard entry found.") + coverages = [ + outcome.scores.coverage + for outcome in scored + if outcome.scores is not None and outcome.scores.coverage is not None + ] + if coverages: + mean_coverage = sum(coverages) / len(coverages) + lines.append( + f"- Mean coverage: {mean_coverage:.2f} (over {len(coverages)} scored questions)" + ) + return lines + + +def build_summary(table: OutcomeTable, top_n: int = 10) -> str: + """ + Render an outcome table as a markdown report. + + :param table: the questions and standing to report on + :param top_n: how many questions to list under best and worst + """ + questions = table.questions + forecasted = [outcome for outcome in questions if outcome.forecasted] + scored = [outcome for outcome in forecasted if outcome.scores is not None] + unscored = [outcome for outcome in forecasted if outcome.scores is None] + resolved_unscored = [ + outcome for outcome in unscored if outcome.status == "resolved" + ] + ranked = sorted( + ( + outcome + for outcome in scored + if outcome.scores is not None + and getattr(outcome.scores, RANK_METRIC) is not None + ), + key=lambda outcome: getattr(outcome.scores, RANK_METRIC), + reverse=True, + ) + + lines = [ + f"# Review: {table.project_name or 'forecast outcomes'}", + "", + f"user {table.user_id} — generated {table.generated_at.isoformat()}", + "", + ] + lines += _overall_lines(table, scored) + lines += [ + "", + "## Participation", + "", + f"- Questions: {len(questions)}", + f"- Forecasted: {len(forecasted)}", + f"- Scored: {len(scored)}", + ( + f"- Forecasted but unscored: {len(unscored)} " + f"({len(resolved_unscored)} resolved without a score)" + ), + ] + for outcome in resolved_unscored: + lines.append( + f" - [{outcome.title}]({outcome.url}) — resolution: {outcome.resolution}" + ) + + lines += ["", "### By tournament", ""] + for label, count in Counter(_project_label(o) for o in questions).most_common(): + scored_here = sum(1 for o in scored if _project_label(o) == label) + lines.append(f"- {label}: {count} questions ({scored_here} scored)") + + lines += ["", f"## Best {min(top_n, len(ranked))} (by spot peer score)", ""] + lines += [_question_line(i, o) for i, o in enumerate(ranked[:top_n], 1)] + lines += ["", f"## Worst {min(top_n, len(ranked))} (by spot peer score)", ""] + lines += [_question_line(i, o) for i, o in enumerate(reversed(ranked[-top_n:]), 1)] + + return "\n".join(lines) + "\n" From 0127749013cc3a798d5d577e0028b3133eaf5261 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:24:41 +0100 Subject: [PATCH 13/21] Let the review render from a saved table instead of refetching The list endpoint has no my_forecasts, so building a table costs one request per post. --from-json re-renders a table saved with --output without touching the API. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_summary.py | 6 ++++ forecasting_tools/bot_review/cli.py | 35 +++++++++++++------ 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/code_tests/unit_tests/test_bot_review/test_summary.py b/code_tests/unit_tests/test_bot_review/test_summary.py index bf8daab8..3fb5883e 100644 --- a/code_tests/unit_tests/test_bot_review/test_summary.py +++ b/code_tests/unit_tests/test_bot_review/test_summary.py @@ -165,6 +165,12 @@ def test_questions_without_a_spot_peer_score_are_not_ranked(): assert "Q1" not in report.split("## Best")[1] +def test_a_table_saved_to_json_renders_the_same_report(): + table = make_table() + reloaded = OutcomeTable.model_validate_json(table.model_dump_json()) + assert build_summary(reloaded) == build_summary(table) + + def test_no_scored_questions(): questions = [make_question(1, scored=False, status="open")] report = build_summary(make_table(questions=questions, leaderboard=None)) diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index 76983cc1..2cfdffdb 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -18,6 +18,19 @@ from forecasting_tools.helpers.metaculus_client import MetaculusClient +def _write_and_print(table: OutcomeTable, args: argparse.Namespace) -> None: + report = build_summary(table, top_n=args.top) + print(report) + if args.output: + with open(args.output, "w") as file: + file.write(table.model_dump_json(indent=2)) + print(f"wrote {args.output}") + if args.summary: + with open(args.summary, "w") as file: + file.write(report) + print(f"wrote {args.summary}") + + def main() -> None: parser = argparse.ArgumentParser(description="Review how a bot's forecasts did") source = parser.add_mutually_exclusive_group(required=True) @@ -29,6 +42,9 @@ def main() -> None: metavar="DAYS", help="your questions that resolved in the last DAYS days, any tournament", ) + source.add_argument( + "--from-json", metavar="FILE", help="re-render a table saved with --output" + ) parser.add_argument( "--include-unforecasted", action="store_true", @@ -49,6 +65,13 @@ def main() -> None: logging.basicConfig(level=logging.WARNING) load_dotenv() + if args.from_json: + with open(args.from_json) as file: + table = OutcomeTable.model_validate_json(file.read()) + print(f"\nreviewing forecasts made by user {table.user_id}") + _write_and_print(table, args) + return + client = MetaculusClient( sleep_seconds_between_requests=args.seconds_between_requests ) @@ -74,17 +97,7 @@ def main() -> None: project_name=project_name, ) - report = build_summary(table, top_n=args.top) - print(report) - - if args.output: - with open(args.output, "w") as file: - file.write(table.model_dump_json(indent=2)) - print(f"wrote {args.output}") - if args.summary: - with open(args.summary, "w") as file: - file.write(report) - print(f"wrote {args.summary}") + _write_and_print(table, args) if __name__ == "__main__": From 5a925fdc4587b141679e830a2e91b9bed29391a3 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Sun, 26 Jul 2026 10:30:04 +0100 Subject: [PATCH 14/21] Rank review questions by the leaderboard's own score type The best/worst lists were hardcoded to spot peer score. That is right for MiniBench and the AI benchmark tournaments, whose leaderboards are spot_peer_tournament, but the Metaculus Cup is peer_tournament: a time-averaged score diluted by coverage, so a question can rank very differently under the two. Verified on the MiniBench leaderboard that the entry score is the plain sum of per-question spot peer scores (348.50 over 36 questions), and that peer/spot_peer tracks coverage (corr 0.92). Unknown score types keep the spot peer default. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_summary.py | 41 ++++++++++++++++--- forecasting_tools/bot_review/summary.py | 39 ++++++++++++++---- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/code_tests/unit_tests/test_bot_review/test_summary.py b/code_tests/unit_tests/test_bot_review/test_summary.py index 3fb5883e..964b4830 100644 --- a/code_tests/unit_tests/test_bot_review/test_summary.py +++ b/code_tests/unit_tests/test_bot_review/test_summary.py @@ -15,6 +15,7 @@ def make_question( post_id: int, spot_peer: float | None = 0.0, baseline: float | None = 0.0, + peer: float | None = None, scored: bool = True, forecasted: bool = True, resolution: str | None = "yes", @@ -37,7 +38,10 @@ def make_question( forecast_time=None, scores=( QuestionScores( - spot_peer_score=spot_peer, baseline_score=baseline, coverage=0.5 + spot_peer_score=spot_peer, + baseline_score=baseline, + peer_score=peer, + coverage=0.5, ) if scored else None @@ -48,7 +52,9 @@ def make_question( ) -def make_leaderboard(rank: int = 3, entries: int = 50) -> Leaderboard: +def make_leaderboard( + rank: int = 3, entries: int = 50, score_type: str = "spot_peer_tournament" +) -> Leaderboard: def entry(user_id: int | None, rank: int | None) -> LeaderboardEntry: return LeaderboardEntry( user_id=user_id, @@ -67,7 +73,7 @@ def entry(user_id: int | None, rank: int | None) -> LeaderboardEntry: project_id=1, project_name="TestCup", project_slug="testcup", - score_type="spot_peer_tournament", + score_type=score_type, finalized=False, entries=[entry(i, i) for i in range(1, entries + 1)], user_entry=entry(7, rank), @@ -135,11 +141,36 @@ def test_best_worst_ordering(): def test_signed_formatting_and_leaderboard(): report = build_summary(make_table()) - assert "+25.0 spot-peer" in report - assert "-15.0 spot-peer" in report + assert "+25.0 spot peer" in report + assert "-15.0 spot peer" in report assert "Rank: **3 / 50** (spot_peer_tournament)" in report +def test_a_peer_tournament_is_ranked_on_peer_score(): + questions = [ + make_question(1, spot_peer=90.0, peer=-5.0), + make_question(2, spot_peer=-90.0, peer=10.0), + ] + report = build_summary( + make_table( + questions=questions, + leaderboard=make_leaderboard(score_type="peer_tournament"), + ) + ) + assert "## Best 2 (by peer score)" in report + assert report.split("## Best")[1].index("Q2") < report.split("## Best")[1].index( + "Q1" + ) + assert "+10.0 peer" in report + + +def test_an_unknown_score_type_falls_back_to_spot_peer(): + report = build_summary( + make_table(leaderboard=make_leaderboard(score_type="comment_insight")) + ) + assert "## Best 4 (by spot peer score)" in report + + def test_mean_coverage_over_scored(): report = build_summary(make_table()) assert "Mean coverage: 0.50 (over 4 scored questions)" in report diff --git a/forecasting_tools/bot_review/summary.py b/forecasting_tools/bot_review/summary.py index fcc4d27a..1abc1bac 100644 --- a/forecasting_tools/bot_review/summary.py +++ b/forecasting_tools/bot_review/summary.py @@ -10,7 +10,24 @@ from forecasting_tools.bot_review.outcomes import OutcomeTable, QuestionOutcome -RANK_METRIC = "spot_peer_score" +METRIC_BY_SCORE_TYPE = { + "peer_tournament": "peer_score", + "spot_peer_tournament": "spot_peer_score", + "spot_baseline_tournament": "spot_baseline_score", + "relative_legacy_tournament": "relative_legacy_score", +} +DEFAULT_METRIC = "spot_peer_score" + + +def _rank_metric(table: OutcomeTable) -> str: + """The per-question score the table's leaderboard ranks on.""" + if table.leaderboard is None: + return DEFAULT_METRIC + return METRIC_BY_SCORE_TYPE.get(table.leaderboard.score_type, DEFAULT_METRIC) + + +def _metric_label(metric: str) -> str: + return metric.removesuffix("_score").replace("_", " ") def _signed(value: float | None) -> str: @@ -21,10 +38,10 @@ def _project_label(outcome: QuestionOutcome) -> str: return outcome.project_name or outcome.project_slug or str(outcome.project_id) -def _question_line(rank: int, outcome: QuestionOutcome) -> str: +def _question_line(rank: int, outcome: QuestionOutcome, metric: str) -> str: assert outcome.scores is not None return ( - f"{rank}. {_signed(outcome.scores.spot_peer_score)} spot-peer · " + f"{rank}. {_signed(getattr(outcome.scores, metric))} {_metric_label(metric)} · " f"{_signed(outcome.scores.baseline_score)} baseline · " f"{outcome.question_type} · [{outcome.title}]({outcome.url})" ) @@ -76,14 +93,15 @@ def build_summary(table: OutcomeTable, top_n: int = 10) -> str: resolved_unscored = [ outcome for outcome in unscored if outcome.status == "resolved" ] + metric = _rank_metric(table) ranked = sorted( ( outcome for outcome in scored if outcome.scores is not None - and getattr(outcome.scores, RANK_METRIC) is not None + and getattr(outcome.scores, metric) is not None ), - key=lambda outcome: getattr(outcome.scores, RANK_METRIC), + key=lambda outcome: getattr(outcome.scores, metric), reverse=True, ) @@ -116,9 +134,12 @@ def build_summary(table: OutcomeTable, top_n: int = 10) -> str: scored_here = sum(1 for o in scored if _project_label(o) == label) lines.append(f"- {label}: {count} questions ({scored_here} scored)") - lines += ["", f"## Best {min(top_n, len(ranked))} (by spot peer score)", ""] - lines += [_question_line(i, o) for i, o in enumerate(ranked[:top_n], 1)] - lines += ["", f"## Worst {min(top_n, len(ranked))} (by spot peer score)", ""] - lines += [_question_line(i, o) for i, o in enumerate(reversed(ranked[-top_n:]), 1)] + metric_label = f"{_metric_label(metric)} score" + lines += ["", f"## Best {min(top_n, len(ranked))} (by {metric_label})", ""] + lines += [_question_line(i, o, metric) for i, o in enumerate(ranked[:top_n], 1)] + lines += ["", f"## Worst {min(top_n, len(ranked))} (by {metric_label})", ""] + lines += [ + _question_line(i, o, metric) for i, o in enumerate(reversed(ranked[-top_n:]), 1) + ] return "\n".join(lines) + "\n" From d3c569723fe23cc69967c24ec24a105068adc422 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Sun, 26 Jul 2026 10:39:58 +0100 Subject: [PATCH 15/21] Attach the bot's own comments to the outcome table as run traces The comment a bot posts is its report, so it doubles as the trace of that run, and it needs no setup: the template ships with folder_to_save_reports_to=None, so local report files usually do not exist, while publish_reports_to_metaculus is on. Measured against the template bot's saved reports, a comment matches the local explanation to within three characters of trailing whitespace. A question can carry several runs, so QuestionOutcome.trace picks the one standing at spot_scoring_time - the run that actually earned the score. The count of questions whose only runs came after that time is reported in the summary, since those forecasts scored nothing under spot scoring. Comments attach to posts, so on a post with more than one question the runs are split by the *Question* line the report writes into its summary. Co-Authored-By: Claude Opus 5 --- .../unit_tests/test_bot_review/test_traces.py | 216 ++++++++++++++++++ forecasting_tools/bot_review/cli.py | 21 ++ forecasting_tools/bot_review/outcomes.py | 23 ++ forecasting_tools/bot_review/summary.py | 5 + forecasting_tools/bot_review/traces.py | 92 ++++++++ 5 files changed, 357 insertions(+) create mode 100644 code_tests/unit_tests/test_bot_review/test_traces.py create mode 100644 forecasting_tools/bot_review/traces.py diff --git a/code_tests/unit_tests/test_bot_review/test_traces.py b/code_tests/unit_tests/test_bot_review/test_traces.py new file mode 100644 index 00000000..cb03fd05 --- /dev/null +++ b/code_tests/unit_tests/test_bot_review/test_traces.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock + +from forecasting_tools.bot_review.outcomes import OutcomeTable, QuestionOutcome +from forecasting_tools.bot_review.traces import attach_traces, get_trace, reduce_comment +from forecasting_tools.data_models.comment import Comment + +RUN_TIME = datetime(2026, 6, 30, 10, tzinfo=timezone.utc) + +EXPLANATION = """ +# SUMMARY +*Question*: Q1 +*Final Prediction*: 20.0% +*Total Cost*: $0.0706 (estimated) +*Time Spent*: 0.28 minutes +*Bot Name*: TemplateBot + +## Report 1 Summary +### Forecasts +*Forecaster 1*: 20.0% +*Forecaster 2 (gpt-5)*: 30.0% + +# RESEARCH +## Report 1 Research +the research + +# FORECASTS +## R1: Forecaster 1 Reasoning +the first rationale +## R1: Forecaster 2 Reasoning +the second rationale +""" + + +def make_comment( + comment_id: int = 1, + post_id: int = 1, + created_at: datetime = RUN_TIME, + text: str = EXPLANATION, +) -> Comment: + return Comment( + id=comment_id, + on_post=post_id, + author_id=7, + author_username="bot", + created_at=created_at, + text=text, + is_private=True, + ) + + +def make_question(post_id: int = 1, title: str = "Q1") -> QuestionOutcome: + return QuestionOutcome( + post_id=post_id, + question_id=post_id + 100, + title=title, + url=f"https://www.metaculus.com/questions/{post_id}/", + question_type="binary", + status="resolved", + resolution="yes", + actual_resolve_time=None, + spot_scoring_time=None, + options=None, + forecasted=True, + forecast={"probability_yes": 0.2}, + forecast_time=None, + scores=None, + project_id=1, + project_slug="testcup", + project_name="TestCup", + ) + + +def make_table(questions) -> OutcomeTable: + return OutcomeTable( + generated_at=RUN_TIME, user_id=7, questions=questions, project_name="TestCup" + ) + + +class TestReduceComment: + def test_reads_the_metadata_the_bot_writes_into_its_report(self): + trace = reduce_comment(make_comment()) + assert trace.comment_id == 1 + assert trace.post_id == 1 + assert trace.run_time == RUN_TIME + assert trace.question_text == "Q1" + assert trace.cost == 0.0706 + assert trace.minutes == 0.28 + assert not trace.truncated + + def test_reads_each_forecaster_prediction(self): + trace = reduce_comment(make_comment()) + assert [f["key"] for f in trace.forecasters] == ["R1:F1", "R1:F2"] + assert [f["model"] for f in trace.forecasters] == [None, "gpt-5"] + assert trace.forecasters[1]["prediction"] == "30.0%" + + def test_missing_metadata_is_none_not_an_error(self): + trace = reduce_comment(make_comment(text="# SUMMARY\nnothing useful here")) + assert trace.cost is None + assert trace.minutes is None + assert trace.question_text is None + assert trace.forecasters == [] + + def test_a_truncated_comment_is_flagged(self): + text = "# SUMMARY\n\n---\n\n The comment size exceeded max size and has been truncated" + assert reduce_comment(make_comment(text=text)).truncated + + +class TestAttachTraces: + def client_returning(self, *comments) -> MagicMock: + client = MagicMock() + client.get_own_comments.return_value = list(comments) + return client + + def test_one_request_per_post_not_per_question(self): + questions = [make_question(1), make_question(1)] + client = self.client_returning(make_comment()) + attach_traces(make_table(questions), client) + assert client.get_own_comments.call_count == 1 + + def test_a_group_post_splits_its_comments_by_question_text(self): + first = make_question(title="Q1") + second = make_question(title="other") + client = self.client_returning( + make_comment(comment_id=1), + make_comment( + comment_id=2, + text=EXPLANATION.replace("*Question*: Q1", "*Question*: other"), + ), + ) + attach_traces(make_table([first, second]), client) + assert [t.comment_id for t in first.traces] == [1] + assert [t.comment_id for t in second.traces] == [2] + + def test_a_single_question_post_keeps_every_run(self): + question = make_question(1) + client = self.client_returning( + make_comment(comment_id=1), make_comment(comment_id=2) + ) + attach_traces(make_table([question]), client) + assert len(question.traces) == 2 + + +class TestSelectingTheScoredRun: + def question_with_runs(self, spot_offset_hours: float | None, *run_offsets: float): + question = make_question(1).model_copy( + update={ + "spot_scoring_time": ( + None + if spot_offset_hours is None + else RUN_TIME + timedelta(hours=spot_offset_hours) + ) + } + ) + question.traces = [ + reduce_comment( + make_comment( + comment_id=i, created_at=RUN_TIME + timedelta(hours=offset) + ) + ) + for i, offset in enumerate(run_offsets) + ] + return question + + def test_the_latest_run_standing_at_spot_time_is_the_one_that_scored(self): + question = self.question_with_runs(5, 0, 2, 9) + assert question.trace is not None + assert question.trace.comment_id == 1 + + def test_a_run_after_spot_time_did_not_earn_the_score(self): + question = self.question_with_runs(5, 9) + assert question.traces + assert question.trace is None + + def test_without_a_spot_time_the_latest_run_is_used(self): + question = self.question_with_runs(None, 0, 9) + assert question.trace is not None + assert question.trace.comment_id == 1 + + def test_no_runs_at_all(self): + assert make_question(1).trace is None + + def test_traces_survive_a_json_round_trip(self): + table = make_table([self.question_with_runs(5, 0)]) + reloaded = OutcomeTable.model_validate_json(table.model_dump_json()) + assert reloaded.questions[0].trace is not None + assert reloaded.questions[0].trace.forecasters[0]["key"] == "R1:F1" + + +class TestGetTrace: + def client_returning(self, *comments) -> MagicMock: + client = MagicMock() + client.get_own_comments.return_value = list(comments) + return client + + def test_returns_the_asked_for_section(self): + client = self.client_returning(make_comment()) + assert "the research" in get_trace(1, "research", client=client) + + def test_returns_one_forecaster_rationale(self): + client = self.client_returning(make_comment()) + text = get_trace(1, forecaster="R1:F2", client=client) + assert "the second rationale" in text + assert "the first rationale" not in text + + def test_reads_the_newest_comment_on_the_post(self): + client = self.client_returning( + make_comment(comment_id=1, text=EXPLANATION.replace("the research", "old")), + make_comment(comment_id=2, created_at=RUN_TIME + timedelta(hours=1)), + ) + assert "the research" in get_trace(1, "research", client=client) + + def test_a_post_with_no_comments(self): + assert get_trace(1, client=self.client_returning()) == "" diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index 2cfdffdb..b617231a 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -15,6 +15,7 @@ get_tournament_outcomes, ) from forecasting_tools.bot_review.summary import build_summary +from forecasting_tools.bot_review.traces import attach_traces, get_trace from forecasting_tools.helpers.metaculus_client import MetaculusClient @@ -45,6 +46,21 @@ def main() -> None: source.add_argument( "--from-json", metavar="FILE", help="re-render a table saved with --output" ) + source.add_argument( + "--show", + type=int, + metavar="POST_ID", + help="print part of the bot's report on one post", + ) + parser.add_argument( + "--section", + default="research", + choices=["summary", "research", "forecasts"], + help="which section --show prints", + ) + parser.add_argument( + "--forecaster", metavar="KEY", help="print one rationale, e.g. R1:F2" + ) parser.add_argument( "--include-unforecasted", action="store_true", @@ -75,6 +91,10 @@ def main() -> None: client = MetaculusClient( sleep_seconds_between_requests=args.seconds_between_requests ) + if args.show: + print(get_trace(args.show, args.section, args.forecaster, client)) + return + user_id = client.get_current_user_id() print(f"\nreviewing forecasts made by user {user_id}") if args.tournament: @@ -97,6 +117,7 @@ def main() -> None: project_name=project_name, ) + attach_traces(table, client) _write_and_print(table, args) diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py index 325d7f22..6e024de5 100644 --- a/forecasting_tools/bot_review/outcomes.py +++ b/forecasting_tools/bot_review/outcomes.py @@ -35,6 +35,19 @@ class QuestionScores(BaseModel): relative_legacy_score: float | None = None +class RunTrace(BaseModel): + """One run of the bot on one question, as recorded in the comment it posted.""" + + comment_id: int + post_id: int + run_time: datetime + question_text: str | None + forecasters: list[dict[str, Any]] + cost: float | None + minutes: float | None + truncated: bool + + class QuestionOutcome(BaseModel): """A question, the bot's forecast on it, and the result.""" @@ -57,11 +70,21 @@ class QuestionOutcome(BaseModel): project_id: int | None project_slug: str | None project_name: str | None + traces: list[RunTrace] = [] @property def was_annulled(self) -> bool: return self.resolution == "annulled" + @property + def trace(self) -> RunTrace | None: + """The run standing when the question was spot scored, if any was.""" + runs = sorted(self.traces, key=lambda run: run.run_time) + if self.spot_scoring_time is None: + return runs[-1] if runs else None + standing = [run for run in runs if run.run_time <= self.spot_scoring_time] + return standing[-1] if standing else None + class OutcomeTable(BaseModel): """A set of questions, the bot's forecasts on them, and its standing.""" diff --git a/forecasting_tools/bot_review/summary.py b/forecasting_tools/bot_review/summary.py index 1abc1bac..7ce93834 100644 --- a/forecasting_tools/bot_review/summary.py +++ b/forecasting_tools/bot_review/summary.py @@ -123,6 +123,11 @@ def build_summary(table: OutcomeTable, top_n: int = 10) -> str: f"- Forecasted but unscored: {len(unscored)} " f"({len(resolved_unscored)} resolved without a score)" ), + ( + f"- Traced: {sum(1 for o in forecasted if o.trace)} " + f"({sum(1 for o in forecasted if o.traces and not o.trace)} ran only after " + "spot scoring time)" + ), ] for outcome in resolved_unscored: lines.append( diff --git a/forecasting_tools/bot_review/traces.py b/forecasting_tools/bot_review/traces.py new file mode 100644 index 00000000..55860742 --- /dev/null +++ b/forecasting_tools/bot_review/traces.py @@ -0,0 +1,92 @@ +"""Attach the bot's own comments to an outcome table as run traces. + +The comment a bot posts is its report, so it is the trace of that run. A +question can have several, one per run; the one that earned the score is the +run standing when the question was spot scored. +""" + +from __future__ import annotations + +import logging +import re +from collections import defaultdict + +from forecasting_tools.bot_review.outcomes import OutcomeTable, RunTrace +from forecasting_tools.bot_review.report_parsing import ( + forecaster_rationales, + parse_forecasters, + split_sections, + was_truncated, +) +from forecasting_tools.data_models.comment import Comment +from forecasting_tools.helpers.metaculus_client import MetaculusClient + +logger = logging.getLogger(__name__) + +COST_PATTERN = re.compile(r"^\*Total Cost\*:\s*\$([\d.]+)", re.MULTILINE) +MINUTES_PATTERN = re.compile(r"^\*Time Spent\*:\s*([\d.]+) minutes", re.MULTILINE) +QUESTION_PATTERN = re.compile(r"^\*Question\*:(.*)$", re.MULTILINE) + + +def reduce_comment(comment: Comment) -> RunTrace: + """Reduce a posted report to the fields a review needs.""" + summary = split_sections(comment.text).get("summary", "") + question = QUESTION_PATTERN.search(comment.text) + cost = COST_PATTERN.search(comment.text) + minutes = MINUTES_PATTERN.search(comment.text) + return RunTrace( + comment_id=comment.id, + post_id=comment.on_post, + run_time=comment.created_at, + question_text=question.group(1).strip() if question else None, + forecasters=parse_forecasters(summary), + cost=float(cost.group(1)) if cost else None, + minutes=float(minutes.group(1)) if minutes else None, + truncated=was_truncated(comment.text), + ) + + +def attach_traces(table: OutcomeTable, client: MetaculusClient | None = None) -> None: + """ + Fill in ``traces`` on every question in the table, one request per post. + + :param table: the table to attach to, modified in place + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + rows_by_post = defaultdict(list) + for outcome in table.questions: + rows_by_post[outcome.post_id].append(outcome) + logger.info(f"Fetching comments for {len(rows_by_post)} posts") + for post_id, rows in rows_by_post.items(): + runs = [reduce_comment(c) for c in client.get_own_comments(post_id=post_id)] + for outcome in rows: + outcome.traces = ( + runs + if len(rows) == 1 + else [run for run in runs if run.question_text == outcome.title] + ) + + +def get_trace( + post_id: int, + section: str = "research", + forecaster: str | None = None, + client: MetaculusClient | None = None, +) -> str: + """ + One part of the bot's latest report on a post. + + :param post_id: post id, as it appears in question urls + :param section: summary, research or forecasts + :param forecaster: a key like ``R1:F2``, to read that rationale instead + :param client: client to fetch with, created from the environment if not given + """ + client = client or MetaculusClient() + comments = client.get_own_comments(post_id=post_id) + if not comments: + return "" + text = max(comments, key=lambda comment: comment.created_at).text + if forecaster: + return forecaster_rationales(text).get(forecaster, "") + return split_sections(text).get(section, "") From 125571cd77e1c4296f6a416cbdacc0e40165bee5 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Sun, 26 Jul 2026 10:47:04 +0100 Subject: [PATCH 16/21] Read comments in bulk, and read public ones too get_own_comments defaults to private comments, but bots publish either way: prior_louis_bot's tournament reports are all public (64 comments over 64 posts) while its ad-hoc runs are private (16 over 11), and the template bot's are private. Fetching only one kind silently produced no traces for half the bots that would use this. Reading both per post would have doubled an already per-post request count, so both are now read in bulk and grouped by post: a 37-question tournament went from 37 comment requests to 2. Comments on posts outside the table are dropped without parsing them. Also moves the traced count below the annulled list, where the indented entries were reading as if they belonged to it. Co-Authored-By: Claude Opus 5 --- .../unit_tests/test_bot_review/test_traces.py | 37 ++++++++++++++--- forecasting_tools/bot_review/summary.py | 10 ++--- forecasting_tools/bot_review/traces.py | 40 ++++++++++++++++--- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/code_tests/unit_tests/test_bot_review/test_traces.py b/code_tests/unit_tests/test_bot_review/test_traces.py index cb03fd05..5f53d237 100644 --- a/code_tests/unit_tests/test_bot_review/test_traces.py +++ b/code_tests/unit_tests/test_bot_review/test_traces.py @@ -110,15 +110,39 @@ def test_a_truncated_comment_is_flagged(self): class TestAttachTraces: def client_returning(self, *comments) -> MagicMock: + """A client whose comments are all private, as most bots post them.""" client = MagicMock() - client.get_own_comments.return_value = list(comments) + client.get_own_comments.side_effect = lambda **kwargs: ( + list(comments) if kwargs.get("is_private") else [] + ) return client - def test_one_request_per_post_not_per_question(self): - questions = [make_question(1), make_question(1)] + def test_comments_are_read_in_bulk_not_once_per_post(self): + questions = [make_question(1), make_question(2), make_question(3)] client = self.client_returning(make_comment()) attach_traces(make_table(questions), client) - assert client.get_own_comments.call_count == 1 + assert client.get_own_comments.call_count == 2 + assert [ + c.kwargs["is_private"] for c in client.get_own_comments.call_args_list + ] == [ + True, + False, + ] + + def test_public_comments_are_read_too(self): + question = make_question(1) + client = MagicMock() + client.get_own_comments.side_effect = lambda **kwargs: ( + [] if kwargs.get("is_private") else [make_comment(comment_id=9)] + ) + attach_traces(make_table([question]), client) + assert [t.comment_id for t in question.traces] == [9] + + def test_comments_on_other_posts_are_ignored(self): + question = make_question(1) + client = self.client_returning(make_comment(post_id=1), make_comment(post_id=2)) + attach_traces(make_table([question]), client) + assert len(question.traces) == 1 def test_a_group_post_splits_its_comments_by_question_text(self): first = make_question(title="Q1") @@ -191,8 +215,11 @@ def test_traces_survive_a_json_round_trip(self): class TestGetTrace: def client_returning(self, *comments) -> MagicMock: + """A client whose comments are all private, as most bots post them.""" client = MagicMock() - client.get_own_comments.return_value = list(comments) + client.get_own_comments.side_effect = lambda **kwargs: ( + list(comments) if kwargs.get("is_private") else [] + ) return client def test_returns_the_asked_for_section(self): diff --git a/forecasting_tools/bot_review/summary.py b/forecasting_tools/bot_review/summary.py index 7ce93834..4c00424f 100644 --- a/forecasting_tools/bot_review/summary.py +++ b/forecasting_tools/bot_review/summary.py @@ -123,16 +123,16 @@ def build_summary(table: OutcomeTable, top_n: int = 10) -> str: f"- Forecasted but unscored: {len(unscored)} " f"({len(resolved_unscored)} resolved without a score)" ), - ( - f"- Traced: {sum(1 for o in forecasted if o.trace)} " - f"({sum(1 for o in forecasted if o.traces and not o.trace)} ran only after " - "spot scoring time)" - ), ] for outcome in resolved_unscored: lines.append( f" - [{outcome.title}]({outcome.url}) — resolution: {outcome.resolution}" ) + lines.append( + f"- Traced: {sum(1 for o in forecasted if o.trace)} " + f"({sum(1 for o in forecasted if o.traces and not o.trace)} ran only after " + "spot scoring time)" + ) lines += ["", "### By tournament", ""] for label, count in Counter(_project_label(o) for o in questions).most_common(): diff --git a/forecasting_tools/bot_review/traces.py b/forecasting_tools/bot_review/traces.py index 55860742..3c601a79 100644 --- a/forecasting_tools/bot_review/traces.py +++ b/forecasting_tools/bot_review/traces.py @@ -46,20 +46,43 @@ def reduce_comment(comment: Comment) -> RunTrace: ) -def attach_traces(table: OutcomeTable, client: MetaculusClient | None = None) -> None: +def attach_traces( + table: OutcomeTable, + client: MetaculusClient | None = None, + max_comments: int = 500, +) -> None: """ - Fill in ``traces`` on every question in the table, one request per post. + Fill in ``traces`` on every question in the table. + + Bots publish their reports privately or publicly depending on how they are + configured, so both are fetched. :param table: the table to attach to, modified in place :param client: client to fetch with, created from the environment if not given + :param max_comments: how far back to page, per privacy setting """ client = client or MetaculusClient() + user_id = client.get_current_user_id() + comments = [ + comment + for is_private in (True, False) + for comment in client.get_own_comments( + is_private=is_private, user_id=user_id, max_comments=max_comments + ) + ] rows_by_post = defaultdict(list) for outcome in table.questions: rows_by_post[outcome.post_id].append(outcome) - logger.info(f"Fetching comments for {len(rows_by_post)} posts") + comments_by_post = defaultdict(list) + for comment in comments: + if comment.on_post in rows_by_post: + comments_by_post[comment.on_post].append(comment) + logger.info( + f"Read {len(comments)} comments, {sum(map(len, comments_by_post.values()))} " + f"of them on the {len(rows_by_post)} posts in the table" + ) for post_id, rows in rows_by_post.items(): - runs = [reduce_comment(c) for c in client.get_own_comments(post_id=post_id)] + runs = [reduce_comment(c) for c in comments_by_post[post_id]] for outcome in rows: outcome.traces = ( runs @@ -83,7 +106,14 @@ def get_trace( :param client: client to fetch with, created from the environment if not given """ client = client or MetaculusClient() - comments = client.get_own_comments(post_id=post_id) + user_id = client.get_current_user_id() + comments = [ + comment + for is_private in (True, False) + for comment in client.get_own_comments( + post_id=post_id, is_private=is_private, user_id=user_id + ) + ] if not comments: return "" text = max(comments, key=lambda comment: comment.created_at).text From 55ad48e62ace9a2cf74730c983f16a95cf4cf7da Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:04:34 +0100 Subject: [PATCH 17/21] Point a deep dive at the run that scored, not the latest one A trace records which run was standing when the question was spot scored, but get_trace read whichever comment was newest, so on a question the bot re-forecast it would have handed back reasoning that had no effect on the score. It now takes an optional comment_id, exposed as --comment, and the trace's own comment_id is what to pass. Drops post_id from RunTrace: it duplicated the row that owns the trace and nothing read it. Co-Authored-By: Claude Opus 5 --- .../unit_tests/test_bot_review/test_traces.py | 14 +++++++++++++- forecasting_tools/bot_review/cli.py | 8 +++++++- forecasting_tools/bot_review/outcomes.py | 1 - forecasting_tools/bot_review/traces.py | 6 ++++-- 4 files changed, 24 insertions(+), 5 deletions(-) diff --git a/code_tests/unit_tests/test_bot_review/test_traces.py b/code_tests/unit_tests/test_bot_review/test_traces.py index 5f53d237..b9d8a3c1 100644 --- a/code_tests/unit_tests/test_bot_review/test_traces.py +++ b/code_tests/unit_tests/test_bot_review/test_traces.py @@ -83,7 +83,6 @@ class TestReduceComment: def test_reads_the_metadata_the_bot_writes_into_its_report(self): trace = reduce_comment(make_comment()) assert trace.comment_id == 1 - assert trace.post_id == 1 assert trace.run_time == RUN_TIME assert trace.question_text == "Q1" assert trace.cost == 0.0706 @@ -239,5 +238,18 @@ def test_reads_the_newest_comment_on_the_post(self): ) assert "the research" in get_trace(1, "research", client=client) + def test_reads_the_run_that_scored_when_given_its_comment_id(self): + client = self.client_returning( + make_comment( + comment_id=1, text=EXPLANATION.replace("the research", "the scored run") + ), + make_comment(comment_id=2, created_at=RUN_TIME + timedelta(hours=1)), + ) + assert "the scored run" in get_trace(1, "research", comment_id=1, client=client) + + def test_an_unknown_comment_id_returns_nothing(self): + client = self.client_returning(make_comment(comment_id=1)) + assert get_trace(1, comment_id=99, client=client) == "" + def test_a_post_with_no_comments(self): assert get_trace(1, client=self.client_returning()) == "" diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index b617231a..ea62b352 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -61,6 +61,12 @@ def main() -> None: parser.add_argument( "--forecaster", metavar="KEY", help="print one rationale, e.g. R1:F2" ) + parser.add_argument( + "--comment", + type=int, + metavar="ID", + help="which run --show reads, from a trace's comment_id (default: the latest)", + ) parser.add_argument( "--include-unforecasted", action="store_true", @@ -92,7 +98,7 @@ def main() -> None: sleep_seconds_between_requests=args.seconds_between_requests ) if args.show: - print(get_trace(args.show, args.section, args.forecaster, client)) + print(get_trace(args.show, args.section, args.forecaster, args.comment, client)) return user_id = client.get_current_user_id() diff --git a/forecasting_tools/bot_review/outcomes.py b/forecasting_tools/bot_review/outcomes.py index 6e024de5..1e1e0391 100644 --- a/forecasting_tools/bot_review/outcomes.py +++ b/forecasting_tools/bot_review/outcomes.py @@ -39,7 +39,6 @@ class RunTrace(BaseModel): """One run of the bot on one question, as recorded in the comment it posted.""" comment_id: int - post_id: int run_time: datetime question_text: str | None forecasters: list[dict[str, Any]] diff --git a/forecasting_tools/bot_review/traces.py b/forecasting_tools/bot_review/traces.py index 3c601a79..f778c554 100644 --- a/forecasting_tools/bot_review/traces.py +++ b/forecasting_tools/bot_review/traces.py @@ -36,7 +36,6 @@ def reduce_comment(comment: Comment) -> RunTrace: minutes = MINUTES_PATTERN.search(comment.text) return RunTrace( comment_id=comment.id, - post_id=comment.on_post, run_time=comment.created_at, question_text=question.group(1).strip() if question else None, forecasters=parse_forecasters(summary), @@ -95,14 +94,16 @@ def get_trace( post_id: int, section: str = "research", forecaster: str | None = None, + comment_id: int | None = None, client: MetaculusClient | None = None, ) -> str: """ - One part of the bot's latest report on a post. + One part of a report the bot posted, by default its latest on the post. :param post_id: post id, as it appears in question urls :param section: summary, research or forecasts :param forecaster: a key like ``R1:F2``, to read that rationale instead + :param comment_id: a trace's ``comment_id``, to read the run that scored :param client: client to fetch with, created from the environment if not given """ client = client or MetaculusClient() @@ -113,6 +114,7 @@ def get_trace( for comment in client.get_own_comments( post_id=post_id, is_private=is_private, user_id=user_id ) + if comment_id is None or comment.id == comment_id ] if not comments: return "" From 8da6fe590ca02de2fccc7f588c86a095a751d7a8 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:54:32 +0100 Subject: [PATCH 18/21] Document the bot-review CLI in the README Covers the four ways to build a table, what the summary and the JSON each carry, pulling reasoning back out a section at a time, and the Python equivalent. Verified the documented calls against a live question. Co-Authored-By: Claude Opus 5 --- README.md | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/README.md b/README.md index 87729114..2e4c7b73 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Here are the tools most likely to be useful to you: - 🎯 **Forecasting Bot:** General forecaster that integrates with the Metaculus AI benchmarking competition and provides a number of utilities. You can forecast with a pre-existing bot or override the class to customize your own (without redoing all the aggregation/API code, etc) - 🔌 **Metaculus API Wrapper:** for interacting with questions and tournaments - 🤖 **In-House Metaculus Bots**: You can see all the bots that Metaculus is running on their site in `run_bots.py` +- 🔍 **Bot Review:** `bot-review` scores your bot's resolved forecasts and shows what it was thinking on the ones it got wrong. See [Reviewing a bot's forecasts](#reviewing-a-bots-forecasts) Here are some other features of the project (not all are documented yet): - **General LLM Wrapper:** A unified interface around litellm with retry logic, the Metaculus proxy, structured outputs, and cost tracking @@ -511,6 +512,49 @@ with MonetaryCostManager(max_cost) as cost_manager: print(f"Current cost: ${current_cost:.2f}") ``` +# Reviewing a bot's forecasts + +`bot-review` builds a table of how your bot's forecasts turned out and attaches the reports it +posted, so you can see what it was thinking on the questions it got wrong. Everything is +read-only: it makes no forecasts, spends nothing on LLMs, and publishes nothing. All it needs +is `METACULUS_TOKEN`. + +```bash +bot-review --tournament --output review.json --summary review.md +bot-review --resolved-since 30 # anything that resolved recently +bot-review --post 44328 44326 # specific questions +bot-review --from-json review.json --top 3 # re-render without refetching +``` + +The markdown summary gives your leaderboard standing, how many questions were forecast and +scored, and the best and worst questions ranked on whichever score the leaderboard uses — spot +peer for the AI benchmark tournaments, time-averaged peer for the Metaculus Cup. + +The JSON adds per-question detail: the official scores, the bot's forecast, and one `RunTrace` +per run taken from the comment that run posted — each forecaster's prediction, the run time, +cost and minutes where the bot leaves that metadata in. `QuestionOutcome.trace` picks the run +that was standing when the question was spot scored, which is the one that earned the score. + +Reasoning text is not stored. Pull it a piece at a time: + +```bash +bot-review --show 44328 --comment 921582 --section research +bot-review --show 44328 --comment 921582 --forecaster R1:F3 +``` + +Or from Python: + +```python +from forecasting_tools.bot_review.outcomes import get_tournament_outcomes +from forecasting_tools.bot_review.summary import build_summary +from forecasting_tools.bot_review.traces import attach_traces, get_trace + +table = get_tournament_outcomes("minibench-2026-06-29") +attach_traces(table) +print(build_summary(table)) +print(get_trace(44328, section="research")) +``` + # Local Development ## Environment Variables From ed490428d70c8415fc20ec9f6488a9a9ff36b443 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Mon, 27 Jul 2026 20:32:44 +0100 Subject: [PATCH 19/21] Drop the model field from parsed forecasters Nothing in forecasting-tools writes a model name into a forecaster bullet (forecast_bot.py emits a bare "*Forecaster N*:"), so the field only ever populated for bots that annotate their own summaries. Reading one bot's convention is exactly what this module is meant to avoid. The parenthesised suffix is still tolerated by the pattern, now non-capturing, so annotated bullets keep parsing rather than failing to match. The forecaster key is the stable identifier across questions. Co-Authored-By: Claude Opus 5 --- .../test_bot_review/test_report_parsing.py | 16 ++++++++-------- .../unit_tests/test_bot_review/test_traces.py | 1 - forecasting_tools/bot_review/report_parsing.py | 10 ++-------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/code_tests/unit_tests/test_bot_review/test_report_parsing.py b/code_tests/unit_tests/test_bot_review/test_report_parsing.py index 5212c002..fa07568f 100644 --- a/code_tests/unit_tests/test_bot_review/test_report_parsing.py +++ b/code_tests/unit_tests/test_bot_review/test_report_parsing.py @@ -55,7 +55,7 @@ R2F2 reasoning. """.strip() -# Bots may annotate the bullets with the model behind each forecaster. +# Some bots add a parenthesised suffix to the bullets; it must not break parsing. ANNOTATED_EXPLANATION = """ # SUMMARY *Question*: Will X happen? @@ -125,17 +125,17 @@ class TestParseForecasters: def test_unannotated_bullets_across_two_reports(self): summary = split_sections(TEMPLATE_EXPLANATION)["summary"] assert parse_forecasters(summary) == [ - {"key": "R1:F1", "model": None, "prediction": "30.0%"}, - {"key": "R1:F2", "model": None, "prediction": "40.0%"}, - {"key": "R2:F1", "model": None, "prediction": "50.0%"}, - {"key": "R2:F2", "model": None, "prediction": "60.0%"}, + {"key": "R1:F1", "prediction": "30.0%"}, + {"key": "R1:F2", "prediction": "40.0%"}, + {"key": "R2:F1", "prediction": "50.0%"}, + {"key": "R2:F2", "prediction": "60.0%"}, ] - def test_model_names_when_the_bot_annotates_them(self): + def test_annotated_bullets_still_parse(self): summary = split_sections(ANNOTATED_EXPLANATION)["summary"] assert parse_forecasters(summary) == [ - {"key": "R1:F1", "model": "a-model", "prediction": "3.0%"}, - {"key": "R1:F2", "model": "another-model", "prediction": "4.0%"}, + {"key": "R1:F1", "prediction": "3.0%"}, + {"key": "R1:F2", "prediction": "4.0%"}, ] def test_research_summary_is_not_read_as_a_prediction(self): diff --git a/code_tests/unit_tests/test_bot_review/test_traces.py b/code_tests/unit_tests/test_bot_review/test_traces.py index b9d8a3c1..48b7a517 100644 --- a/code_tests/unit_tests/test_bot_review/test_traces.py +++ b/code_tests/unit_tests/test_bot_review/test_traces.py @@ -92,7 +92,6 @@ def test_reads_the_metadata_the_bot_writes_into_its_report(self): def test_reads_each_forecaster_prediction(self): trace = reduce_comment(make_comment()) assert [f["key"] for f in trace.forecasters] == ["R1:F1", "R1:F2"] - assert [f["model"] for f in trace.forecasters] == [None, "gpt-5"] assert trace.forecasters[1]["prediction"] == "30.0%" def test_missing_metadata_is_none_not_an_error(self): diff --git a/forecasting_tools/bot_review/report_parsing.py b/forecasting_tools/bot_review/report_parsing.py index 2de6821c..76ed6a2e 100644 --- a/forecasting_tools/bot_review/report_parsing.py +++ b/forecasting_tools/bot_review/report_parsing.py @@ -16,9 +16,7 @@ SECTIONS = ("summary", "research", "forecasts") REPORT_PATTERN = re.compile(r"^## Report (\d+) Summary\s*$", re.MULTILINE) -FORECASTER_PATTERN = re.compile( - r"^\*Forecaster (\d+)(?: \(([^)]*)\))?\*:", re.MULTILINE -) +FORECASTER_PATTERN = re.compile(r"^\*Forecaster (\d+)(?: \([^)]*\))?\*:", re.MULTILINE) RATIONALE_TITLE_PATTERN = re.compile(r"^R(\d+): Forecaster (\d+) Reasoning$") FINAL_PREDICTION_PATTERN = re.compile( r"^\*Final Prediction\*:(.*?)(?=^\*[A-Z]|^## |\Z)", re.MULTILINE | re.DOTALL @@ -66,10 +64,7 @@ def forecasts_block(summary: str) -> str: def parse_forecasters(summary: str) -> list[dict]: - """Each forecaster's prediction as it was written in the summary. - - ``model`` is None unless the bot annotates its summary bullets with model names. - """ + """Each forecaster's prediction as it was written in the summary.""" reports = list(REPORT_PATTERN.finditer(summary)) blocks = _bodies_between(summary, [(m.start(), m.end()) for m in reports]) forecasters = [] @@ -80,7 +75,6 @@ def parse_forecasters(summary: str) -> list[dict]: forecasters += [ { "key": f"R{report.group(1)}:F{match.group(1)}", - "model": match.group(2), "prediction": body, } for match, body in zip(matches, bodies) From fd62b9835c0f05d050178557a26418f8b1d4d447 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:03:47 +0100 Subject: [PATCH 20/21] Split the CLI into review and show subcommands The two modes had disjoint flag sets sharing one namespace, so "--show 44328 --output x.json" ran, printed research, wrote no file and reported nothing. Subparsers make the combination impossible to express. bot-review review --tournament --output review.json bot-review show 44328 --section research Co-Authored-By: Claude Opus 5 --- README.md | 12 ++--- forecasting_tools/bot_review/cli.py | 70 ++++++++++++++++------------- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 2e4c7b73..6d0447cf 100644 --- a/README.md +++ b/README.md @@ -520,10 +520,10 @@ read-only: it makes no forecasts, spends nothing on LLMs, and publishes nothing. is `METACULUS_TOKEN`. ```bash -bot-review --tournament --output review.json --summary review.md -bot-review --resolved-since 30 # anything that resolved recently -bot-review --post 44328 44326 # specific questions -bot-review --from-json review.json --top 3 # re-render without refetching +bot-review review --tournament --output review.json --summary review.md +bot-review review --resolved-since 30 # anything that resolved recently +bot-review review --post 44328 44326 # specific questions +bot-review review --from-json review.json --top 3 # re-render without refetching ``` The markdown summary gives your leaderboard standing, how many questions were forecast and @@ -538,8 +538,8 @@ that was standing when the question was spot scored, which is the one that earne Reasoning text is not stored. Pull it a piece at a time: ```bash -bot-review --show 44328 --comment 921582 --section research -bot-review --show 44328 --comment 921582 --forecaster R1:F3 +bot-review show 44328 --section research +bot-review show 44328 --forecaster R1:F3 --comment 921582 # a specific run ``` Or from Python: diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index ea62b352..32774122 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -34,7 +34,21 @@ def _write_and_print(table: OutcomeTable, args: argparse.Namespace) -> None: def main() -> None: parser = argparse.ArgumentParser(description="Review how a bot's forecasts did") - source = parser.add_mutually_exclusive_group(required=True) + common = argparse.ArgumentParser(add_help=False) + common.add_argument( + "--seconds-between-requests", + type=float, + default=0.7, + help="delay between Metaculus requests", + ) + commands = parser.add_subparsers(dest="command", required=True) + + review = commands.add_parser( + "review", + parents=[common], + help="score the bot's questions and write the summary", + ) + source = review.add_mutually_exclusive_group(required=True) source.add_argument("--tournament", help="tournament slug or id") source.add_argument("--post", type=int, nargs="+", metavar="POST_ID") source.add_argument( @@ -46,48 +60,42 @@ def main() -> None: source.add_argument( "--from-json", metavar="FILE", help="re-render a table saved with --output" ) - source.add_argument( - "--show", - type=int, - metavar="POST_ID", - help="print part of the bot's report on one post", + review.add_argument( + "--include-unforecasted", + action="store_true", + help="also include questions in the tournament that the bot never forecast", ) - parser.add_argument( + review.add_argument("--output", help="write the full table to this json file") + review.add_argument("--summary", help="write the markdown report to this file") + review.add_argument( + "--top", type=int, default=10, help="how many best and worst questions to list" + ) + + show = commands.add_parser( + "show", parents=[common], help="print part of one report the bot posted" + ) + show.add_argument("post_id", type=int) + show.add_argument( "--section", default="research", choices=["summary", "research", "forecasts"], - help="which section --show prints", + help="which section to print", ) - parser.add_argument( + show.add_argument( "--forecaster", metavar="KEY", help="print one rationale, e.g. R1:F2" ) - parser.add_argument( + show.add_argument( "--comment", type=int, metavar="ID", - help="which run --show reads, from a trace's comment_id (default: the latest)", - ) - parser.add_argument( - "--include-unforecasted", - action="store_true", - help="also include questions in the tournament that the bot never forecast", - ) - parser.add_argument("--output", help="write the full table to this json file") - parser.add_argument("--summary", help="write the markdown report to this file") - parser.add_argument( - "--top", type=int, default=10, help="how many best and worst questions to list" - ) - parser.add_argument( - "--seconds-between-requests", - type=float, - default=0.7, - help="delay between Metaculus requests", + help="which run to read, from a trace's comment_id (default: the latest)", ) + args = parser.parse_args() logging.basicConfig(level=logging.WARNING) load_dotenv() - if args.from_json: + if args.command == "review" and args.from_json: with open(args.from_json) as file: table = OutcomeTable.model_validate_json(file.read()) print(f"\nreviewing forecasts made by user {table.user_id}") @@ -97,8 +105,10 @@ def main() -> None: client = MetaculusClient( sleep_seconds_between_requests=args.seconds_between_requests ) - if args.show: - print(get_trace(args.show, args.section, args.forecaster, args.comment, client)) + if args.command == "show": + print( + get_trace(args.post_id, args.section, args.forecaster, args.comment, client) + ) return user_id = client.get_current_user_id() From 02358af5d0991a14485f9d933b9c780b874d10a2 Mon Sep 17 00:00:00 2001 From: Louis Prosser <77465609+LouisP96@users.noreply.github.com> Date: Mon, 27 Jul 2026 21:21:46 +0100 Subject: [PATCH 21/21] Load .env from the working directory, not the caller frame Bare load_dotenv() resolves relative to the frame that calls it, so from an installed console script it walks up from site-packages and finds nothing. "bot-review" is what the README and the workflow tell people to run, and it could not see a project .env at all; only "python -m" worked. Co-Authored-By: Claude Opus 5 --- forecasting_tools/bot_review/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/forecasting_tools/bot_review/cli.py b/forecasting_tools/bot_review/cli.py index 32774122..ff853efe 100644 --- a/forecasting_tools/bot_review/cli.py +++ b/forecasting_tools/bot_review/cli.py @@ -6,7 +6,7 @@ import logging from datetime import datetime, timezone -from dotenv import load_dotenv +from dotenv import find_dotenv, load_dotenv from forecasting_tools.bot_review.outcomes import ( OutcomeTable, @@ -93,7 +93,7 @@ def main() -> None: args = parser.parse_args() logging.basicConfig(level=logging.WARNING) - load_dotenv() + load_dotenv(find_dotenv(usecwd=True)) if args.command == "review" and args.from_json: with open(args.from_json) as file: