diff --git a/.gitignore b/.gitignore index da865b9..204e986 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__ latest.log logs/ sandbox.py -local/ \ No newline at end of file +local/ +temp/ \ No newline at end of file diff --git a/README.md b/README.md index ec0a314..0cd4d44 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,8 @@ -The main entry point to the project is `aib_analysis/main.py` - ## Installing Dependencies -To set up the project please run: -Installing dependencies -Make sure you have python and poetry installed (poetry is a python package manager). -If you don't have poetry installed run the below: +Make sure you have Python and Poetry installed (Poetry is a Python package manager). + +If you don't have Poetry installed: ``` sudo apt update -y sudo apt install -y pipx @@ -15,42 +12,47 @@ pipx install poetry poetry config virtualenvs.in-project true ``` -Inside the terminal, go to the directory you cloned the repository into and run the following command: - +From the repo root: ``` poetry install ``` to install all required dependencies. -You may also need to follow instructions [here](https://git-lfs.com/) for setting up git management for large files. +## Input data + +Forecast CSVs live under `local/private_input_data/` (gitignored). For Spring 2026 the simulation script expects: + +- `local/private_input_data/pro_forecasts_2026_spring.csv` +- `local/private_input_data/bot_forecasts_2026_spring.csv` + +You can: -## Running the Analysis +- If you have access, download prior analysis outputs (and sometimes inputs) from [this Google Drive folder](https://drive.google.com/drive/folders/1m7e8AQd4M-Y4oPuj--dDAEYwxO-J2kth?usp=drive_link), or +- Request access to analysis data from the instructions shared [here](https://www.metaculus.com/notebooks/38928/ai-benchmark-resources/#what-data-do-i-have-access-to-via-api-how-can-i-get-access-to-more) which will show you how to use the [Metaculus Data Needs Form](https://docs.google.com/forms/d/e/1FAIpQLSeJhtZzHl5qMvBjbXbatyaqoS4IU7RE0GGw_vlhs6I9syqn1g/viewform?usp=pp_url&entry.192763438=https://www.metaculus.com/api/) -To generate the analysis data run `aib_analysis/run_q1_q1_simulations.py`. +## Running the analysis -Check the logs for warnings to see edge cases that have come up. +For the current (Spring 2026) tournament: -To display the analysis please execute: ``` -poetry run streamlit run aib_analysis/front_end.py +poetry run python aib_analysis/run_spring_simulations.py ``` -This will bring up the visuals for the analysis. Make sure you have chosen the right input data in this script. Front end uses caching, so restart streamlit if you update the underlying data. +That writes JSON artifacts to `local/spring_2026_simulations_teams_comparison/`. + +Check `logs/latest_info.log` for warnings (annulled questions, weight mismatches, unmatched questions, missing control bots, etc.). +For older seasons (Q1–Q4), use `aib_analysis/run_q1_q2_simulations.py` and edit the `__main__` paths/folders as needed. -Make sure to restart the site or click "clear cache" in the triple dot menu on the site if you change the underlying data or simulated_tournament loading/intialization code. +## Viewing results +``` +poetry run streamlit run aib_analysis/front_end.py +``` -You can see/download/use previous analysis results here https://drive.google.com/drive/folders/1m7e8AQd4M-Y4oPuj--dDAEYwxO-J2kth?usp=drive_link +In the UI, set the tournaments folder to the simulation output path (e.g. `local/spring_2026_simulations_teams_comparison`). Restart Streamlit or use **Clear cache** in the ⋮ menu after changing underlying JSON or loading/scoring code. ## Structure -The project is focused around the SimulatedTournament object. This is initialized with a number of Forecast objects (see `data_models.py`) and used to create other parts of a tournament (Users, Scores, etc). Every data analysis item we care about is just a tournament of some sort. Often this is a filter, aggregation, intersection, etc of forecasts from another tournament, but even a comparison of 2 people is a tournament. - -Refactor advantages -- Pydantic model validation (type safety, and consistency checks at initialization time) -- Easier to unit test -- More reusable (e.g. We can use this structure for regular peer score comparisons in the future) -- Git diffs are easier to read -- We can eventually publish our results as an interactive app if we want to (and if not, we can more easily check and interact with the data ourselves) +The project is built around `SimulatedTournament` (see `data_models.py` / `simulated_tournament.py`). It is initialized from `Forecast` objects and derives users, scores, etc. Most analyses are filtered, aggregated, or combined tournaments built from those forecasts. diff --git a/aib_analysis/data_structures/custom_types.py b/aib_analysis/data_structures/custom_types.py index 7ba0bbc..c5c2d29 100644 --- a/aib_analysis/data_structures/custom_types.py +++ b/aib_analysis/data_structures/custom_types.py @@ -4,13 +4,14 @@ BinaryResolutionType = bool MCResolutionType = str NumericResolutionType = float -ResolutionType = BinaryResolutionType | MCResolutionType | NumericResolutionType | AnnulledAmbiguousResolutionType +DiscreteResolutionType = float +ResolutionType = BinaryResolutionType | MCResolutionType | NumericResolutionType | DiscreteResolutionType | AnnulledAmbiguousResolutionType BinaryForecastType = list[float] # binary: [p_yes, p_no] MCForecastType = list[float] # multiple choice: [p_option_a, p_option_b, p_option_c], NumericForecastType = list[float] # numeric: [p_0, p_1, p_2, ..., p_200] (201 value cdf) -ForecastType = BinaryForecastType | MCForecastType | NumericForecastType | None - +DiscreteForecastType = list[float] # discrete: cdf of variable length +ForecastType = BinaryForecastType | MCForecastType | NumericForecastType | DiscreteForecastType | None class UserType(Enum): PRO = "pro" @@ -18,11 +19,11 @@ class UserType(Enum): CP = "community_prediction" AGGREGATE = "aggregate" - class QuestionType(Enum): BINARY = "binary" MULTIPLE_CHOICE = "multiple_choice" NUMERIC = "numeric" + DISCRETE = "discrete" class ScoreType(Enum): diff --git a/aib_analysis/data_structures/data_models.py b/aib_analysis/data_structures/data_models.py index 475722f..0811ab7 100644 --- a/aib_analysis/data_structures/data_models.py +++ b/aib_analysis/data_structures/data_models.py @@ -3,7 +3,7 @@ import logging import random from datetime import datetime, timedelta, timezone - +from typing import Literal from pydantic import BaseModel, ConfigDict, model_validator from typing_extensions import Self @@ -51,7 +51,6 @@ def id(self) -> str: return self._id def get_spot_baseline_score(self, resolution: ResolutionType) -> Score: - self._error_if_need_zero_point() q = self.question score_value = calculate_baseline_score( forecast=self.prediction, @@ -63,6 +62,7 @@ def get_spot_baseline_score(self, resolution: ResolutionType) -> Score: range_max=q.range_max, open_upper_bound=q.open_upper_bound, open_lower_bound=q.open_lower_bound, + zero_point=q.zero_point, ) return Score( @@ -74,13 +74,9 @@ def get_spot_baseline_score(self, resolution: ResolutionType) -> Score: def get_spot_peer_score( self, resolution: ResolutionType, other_users_forecasts: list[Forecast] ) -> Score: - self._error_if_need_zero_point() + # Metaculus includes the scored forecaster in the geometric mean baseline and + # applies N/(N-1) as a leave-one-out approximation, so self may appear here. other_preds = [f.prediction for f in other_users_forecasts] - users_used_in_scoring = [f.user for f in other_users_forecasts] - if self.user in users_used_in_scoring: - raise ValueError( - "Forecast Author cannot be in other users forecasts list for peer score" - ) q = self.question score_value = calculate_peer_score( forecast=self.prediction, @@ -91,6 +87,7 @@ def get_spot_peer_score( options=list(q.options) if q.options is not None else None, range_min=q.range_min, range_max=q.range_max, + zero_point=q.zero_point, ) return Score( score=score_value, @@ -98,10 +95,6 @@ def get_spot_peer_score( forecast=self, ) - def _error_if_need_zero_point(self) -> None: - if self.question.is_log_scale: - raise NotImplementedError(f"Numeric question {self.question.question_id} has a zero point. Log Scall is currently not supported") - @model_validator(mode="after") def check_prediction_type_matches(self) -> Self: if self.prediction is None: @@ -126,9 +119,9 @@ def check_prediction_type_matches(self) -> Self: raise ValueError( "Prediction must be a list of floats (length >= 2) for multiple choice questions." ) - if abs(sum(self.prediction) - 1) > 1e-6: + if abs(sum(self.prediction) - 1) > 1e-5: raise ValueError( - "Prediction must sum to 1 for multiple choice questions." + f"Prediction must sum to 1 for multiple choice questions, got {sum(self.prediction)} for {self.prediction}" ) elif q_type == QuestionType.NUMERIC: is_list = isinstance(self.prediction, list) @@ -139,6 +132,19 @@ def check_prediction_type_matches(self) -> Self: raise ValueError( "Prediction must be a list of 201 floats for numeric questions." ) + elif q_type == QuestionType.DISCRETE: + if self.question.inbound_outcome_count is None: + raise ValueError("inbound_outcome_count must be set for discrete questions.") + + is_list = isinstance(self.prediction, list) + expected_points = self.question.inbound_outcome_count + 1 + has_expected_points = len(self.prediction) == expected_points + non_float_items = [p for p in self.prediction if not isinstance(p, float)] + all_floats = len(non_float_items) == 0 + if not (is_list and has_expected_points and all_floats): + raise ValueError( + f"Prediction must be a list of {expected_points} floats for discrete questions, got {len(self.prediction) if is_list else 'non-list'}." + ) # verify predictions are between 0 and 1 for p in self.prediction: @@ -207,12 +213,14 @@ class Question(BaseModel, frozen=True): open_upper_bound: bool | None open_lower_bound: bool | None zero_point: float | None = None + inbound_outcome_count: int | None = None weight: float post_id: int created_at: datetime spot_scoring_time: datetime project: str | None = None notes: str | None = None + unscored_resolution_reason: Literal["annulled", "ambiguous", "blank"] | None = None # When resolution is None: "annulled" / "ambiguous" from CSV, or "blank" for null resolution (often deleted posts). None if unknown (e.g. older JSON). model_config = ConfigDict(frozen=True) @property @@ -236,6 +244,10 @@ def check_resolution_type_matches(self) -> Self: self.resolution, float ): raise ValueError("Resolution must be a float for numeric questions.") + if self.type == QuestionType.DISCRETE and not isinstance( + self.resolution, float + ): + raise ValueError("Resolution must be a float for discrete questions.") return self @model_validator(mode="after") @@ -267,6 +279,17 @@ def check_question_type_has_right_fields(self) -> Self: raise ValueError( "Numeric questions must have all bound information (upper_bound, lower_bound, open_upper_bound, open_lower_bound)." ) + if self.type == QuestionType.DISCRETE: + if ( + self.range_max is None + or self.range_min is None + or self.open_upper_bound is None + or self.open_lower_bound is None + or self.inbound_outcome_count is None + ): + raise ValueError( + "Discrete questions must have all bound information (upper_bound, lower_bound, open_upper_bound, open_lower_bound) and inbound_outcome_count." + ) if self.type == QuestionType.MULTIPLE_CHOICE: if not self.options or len(self.options) < 2: raise ValueError( @@ -339,6 +362,7 @@ def get_hash_for_tournament_matching(self) -> str: self.open_upper_bound, self.open_lower_bound, self.zero_point, + self.inbound_outcome_count, spot_scoring_window, ) return str(hash(hash_fields)) @@ -348,12 +372,23 @@ class User(BaseModel): name: str type: UserType aggregated_users: list[User] + is_primary_bot: bool | None = None + exclude_from_aggregations: bool | None = None model_config = ConfigDict(frozen=True) @property def is_metac_bot(self) -> bool: return "metac-" in self.name.lower() or "mf-bot-" in self.name.lower() + @property + def contributes_to_peer_baseline(self) -> bool: + """Match Metaculus exclude_non_primary_bots + blacklist filtering.""" + if self.exclude_from_aggregations is True: + return False # None for exclude_from_aggregations is interpreted as a default of False + if self.type == UserType.BOT and self.is_primary_bot is False: + return False # None for is_primary_bot is interpreted as a default of True + return True + class Leaderboard(BaseModel): entries: list[LeaderboardEntry] diff --git a/aib_analysis/data_structures/problem_questions.py b/aib_analysis/data_structures/problem_questions.py index 58ed1af..bc9a044 100644 --- a/aib_analysis/data_structures/problem_questions.py +++ b/aib_analysis/data_structures/problem_questions.py @@ -20,6 +20,7 @@ class Tournament(Enum): Q1_2025_VS_CUP = "q1_2025_vs_cup" Q2_2025 = "q2_2025" Fall_2025 = "fall_2025" + Spring_2026 = "spring_2026" class ProblemQuestion(BaseModel): @@ -97,6 +98,35 @@ def check_urls(self) -> Self: proposed_action="", tournament=Tournament.Q1_2025_VS_CUP, ), + # Spring 2026 + ProblemQuestion( + question_text="How many commercial aircraft deliveries will Airbus report for March 2026 ?", + question_1_url="https://www.metaculus.com/questions/42109/", + question_2_url="https://www.metaculus.com/questions/42272/", + notes="Duplicate questions in the same tournament. One is discrete, one is numeric. Both resolved to 60.", + proposed_action="Remove from comparison", + tournament=Tournament.Spring_2026, + ), + ProblemQuestion( + question_text="What will be the legal status of an EU PPWR delegated act on recyclability performance classes ... May 1, 2026?", + question_1_url="https://www.metaculus.com/questions/42519/", + question_2_url="https://www.metaculus.com/questions/42329/", + notes="Title mismatch only: pro 'on May 1' vs bot 'before May 1'. Same options and resolution ('No publication').", + proposed_action="Force match", + tournament=Tournament.Spring_2026, + ), + ProblemQuestion( + question_text="How many new civil antitrust merger challenges will the DOJ file against 'Big Tech' firms before May 1, 2026?", + question_1_url="https://www.metaculus.com/questions/42094/", + question_2_url="https://www.metaculus.com/questions/42050/", + notes=( + "Already hash-matches; force-match documents the intended pair. " + "Do not pair with annulled pro duplicate https://www.metaculus.com/questions/42063/ " + "(different inbound_outcome_count / spot window / annulled resolution)." + ), + proposed_action="Force match (exclude annulled 42063)", + tournament=Tournament.Spring_2026, + ), ] class ProblemManager: @@ -131,6 +161,39 @@ def _pair_matches_problem_question_in_list( return False +def _all_unscored_title_match_message(title_matched_questions: list[Question]) -> str: + urls = [question.url for question in title_matched_questions] + reasons = { + question.unscored_resolution_reason for question in title_matched_questions + } + if None in reasons: + return ( + "Matching question titles found, but all resolutions are blank " + f"(annulled or deleted): {urls}" + ) + if reasons == {"annulled"}: + return f"Matching question titles found, but all are annulled: {urls}" + if reasons == {"ambiguous"}: + return f"Matching question titles found, but all are ambiguous: {urls}" + if reasons == {"blank"}: + return ( + "Matching question titles found, but all have blank resolutions " + f"(deleted or unresolved): {urls}" + ) + reason_counts = [] + for reason in sorted(reasons): + count = sum( + 1 + for question in title_matched_questions + if question.unscored_resolution_reason == reason + ) + reason_counts.append(f"{count} {reason}") + return ( + "Matching question titles found, but all resolutions are blank " + f"(annulled or deleted): {urls} ({', '.join(reason_counts)})" + ) + + def title_matched_questions_are_problematic( title_matched_questions: list[Question], log_results: bool ) -> bool: @@ -148,7 +211,7 @@ def title_matched_questions_are_problematic( log_comparison_table = False if len(non_annulled_questions) == 0: - info_message = f"Matching question titles found, but all are annulled: {[q.url for q in title_matched_questions]}" + info_message = _all_unscored_title_match_message(title_matched_questions) elif len(non_annulled_questions) == 1: unique_projects = list(set([q.project for q in title_matched_questions])) if len(unique_projects) > 1: diff --git a/aib_analysis/data_structures/simulated_tournament.py b/aib_analysis/data_structures/simulated_tournament.py index 03f49f2..eb81d11 100644 --- a/aib_analysis/data_structures/simulated_tournament.py +++ b/aib_analysis/data_structures/simulated_tournament.py @@ -47,7 +47,6 @@ def initialize_tournament(self) -> Self: unique_forecasts = list(set(tuple(forecasts))) self.forecasts = unique_forecasts - self._remove_log_scale_questions() self._initialize_spot_forecast_cache() self._initialize_user_and_question_caches() if len(self.scores_cache) == 0: @@ -113,7 +112,7 @@ def question_to_spot_forecasts(self, question_id: int) -> list[Forecast]: self._question_to_spot_forecasts_cache[question_id_to_cache].append( forecast ) - spot_forecasts = self._question_to_spot_forecasts_cache[question_id] + spot_forecasts = self._question_to_spot_forecasts_cache.get(question_id, []) return ( spot_forecasts.copy() ) # Shallow copy (so you don't modify order of original list) @@ -168,30 +167,15 @@ def get_spot_score_for_question_and_user( assert len(scores) == 1, "Expected exactly for question for user if spot score" return scores[0] - def _remove_log_scale_questions(self) -> None: - non_log_scale_forecasts = [ - forecast - for forecast in self.forecasts - if not forecast.question.is_log_scale - ] - if not (len(non_log_scale_forecasts) == len(self.forecasts)): - non_log_scaled_questions = [ - question for question in self.questions if question.is_log_scale - ] - logger.warning( - f"Removed {len(self.forecasts) - len(non_log_scale_forecasts)} log scale forecasts and {len(self.questions) - len(non_log_scaled_questions)} log scale questions" - ) - self.forecasts = non_log_scale_forecasts - def _initialize_spot_forecast_cache(self) -> None: spot_forecasts: dict[tuple[str, int], Forecast] = {} for forecast in self.forecasts: question = forecast.question user_name = forecast.user.name spot_time = question.spot_scoring_time - if forecast.prediction_time >= spot_time: + if forecast.prediction_time > spot_time: continue - if forecast.end_time and forecast.end_time < spot_time: + if forecast.end_time and forecast.end_time <= spot_time: continue key = (user_name, question.question_id) current = spot_forecasts.get(key) @@ -237,13 +221,19 @@ def _calculate_spot_scores_for_forecast( if isinstance(resolution, AnnulledAmbiguousResolutionType): return [] - spot_forecasts_from_others: list[Forecast] = self.question_to_spot_forecasts( + spot_forecasts: list[Forecast] = self.question_to_spot_forecasts( forecast_to_score.question.question_id ) - spot_forecasts_from_others.remove(forecast_to_score) + # Metaculus includes the scored forecaster in the geometric mean, then applies + # N/(N-1). It also excludes non-primary bots and blacklisted users. + peer_baseline_forecasts = [ + forecast + for forecast in spot_forecasts + if forecast.user.contributes_to_peer_baseline + ] spot_peer_score = forecast_to_score.get_spot_peer_score( - resolution, spot_forecasts_from_others + resolution, peer_baseline_forecasts ) spot_baseline_score = forecast_to_score.get_spot_baseline_score(resolution) scores = [spot_peer_score, spot_baseline_score] diff --git a/aib_analysis/dataclip_snapshot.sql b/aib_analysis/dataclip_snapshot.sql new file mode 100644 index 0000000..aae081a --- /dev/null +++ b/aib_analysis/dataclip_snapshot.sql @@ -0,0 +1,67 @@ +SELECT + f.id AS forecast_id, + q.id AS question_id, + qp.post_id, + q.title AS question_title, + q.created_at, -- ?? same as publish time? + f.author_id, + f.probability_yes, + f.probability_yes_per_category, + f.continuous_cdf, + f.start_time AS forecast_timestamp, + f.end_time AS forecast_endtime, + u.username AS forecaster, + u.is_bot, + u.is_primary_bot, + u.exclude_from_aggregations, + q.resolution, + q.scheduled_close_time, + q.actual_close_time, + q.scheduled_resolve_time, + q.actual_resolve_time, + q.spot_scoring_time, + p.published_at, + q.question_weight, + q.cp_reveal_time, + q.open_time, + -- q.resolution_criteria, + -- q.description, + -- q.fine_print, + q.type, + q.options, + q.range_min, + q.range_max, + q.open_lower_bound, + q.open_upper_bound, + q.zero_point, + q.inbound_outcome_count, + pr.name as project_title +FROM questions_forecast f +LEFT JOIN users_user u ON f.author_id = u.id +LEFT JOIN questions_question q ON f.question_id = q.id +LEFT JOIN questions_question_post qp ON q.id = qp.question_id +LEFT JOIN posts_post p ON p.id = qp.post_id +LEFT JOIN projects_project pr ON p.default_project_id = pr.id +WHERE q.title NOT ILIKE '%[practice]%' +-- AND p.default_project_id = 3349 -- Q3 AIB BOT Tournament +-- AND p.default_project_id = 32506 -- Q4 AIB BOT Tournament +-- AND p.default_project_id = 32627 -- Q1 AIB BOT Tournament +-- AND p.default_project_id = 32721 -- Q2 AIB BOT Tournament +-- AND p.default_project_id = 32813 -- Fall 2025 AIB BOT Tournament +-- AND p.default_project_id = 32916 -- Spring 2026 FutureEval Bot Tournament +-- AND p.default_project_id = 33022 -- Summer 2026 FutureEval Bot Tournament +-- AND p.default_project_id = 3345 -- Q3 AIB PRO Tournament +-- AND p.default_project_id = 3673 -- Q4 AIB PRO Tournament +-- AND p.default_project_id = 32631 -- Q1 AIB PRO Tournament +-- AND p.default_project_id = 32761 -- Q2 AIB PRO Tournament +AND p.default_project_id = 32930 -- Pro Forecasters - AI Forecasting Benchmark 2026 Spring +-- AND p.default_project_id = 33025 -- Pro Forecasters - FutureEval 2026 Summer +-- AND p.default_project_id = 32827 -- MiniBench sept 1 +-- AND p.default_project_id = 32830 -- MiniBench Sept 15 +--AND p.default_project_id = 32630 -- Q1 Quarterly Cup +-- AND p.default_project_id = 32726 -- Summer Metaculus Cup +-- Do not filter on u.is_bot: include stray humans on bot projects (peer GM parity) and +-- coherence bots on pro projects. Streamlit hides the out-of-scope users from +-- leaderboard charts/tables while keeping them in scoring. +-- AND f.start_time < q.cp_reveal_time -- If for quarterly cup +ORDER BY f.start_time DESC; \ No newline at end of file diff --git a/aib_analysis/main_logic/load_tournament.py b/aib_analysis/main_logic/load_tournament.py index a07398c..8a564e6 100644 --- a/aib_analysis/main_logic/load_tournament.py +++ b/aib_analysis/main_logic/load_tournament.py @@ -97,23 +97,18 @@ def _parse_forecast_row( ) -> tuple[Forecast, Question, User]: prediction = _parse_forecast(row) resolution = _parse_resolution(row) + unscored_resolution_reason = _parse_unscored_resolution_reason(row) question_id = int(row["question_id"]) username = row["forecaster"] if question_id in question_cache: question = question_cache[question_id] else: - cp_reveal_exists = pd.notnull(row.get("cp_reveal_time")) - assert isinstance(cp_reveal_exists, bool) question = Question( question_text=row["question_title"], resolution=resolution, weight=float(row["question_weight"]), - spot_scoring_time=( - pd.to_datetime(row["cp_reveal_time"]) - if cp_reveal_exists - else pd.to_datetime(row["scheduled_close_time"]) - ), + spot_scoring_time=_resolve_spot_scoring_time(row), question_id=question_id, post_id=int(row["post_id"]), type=QuestionType(row["type"]), @@ -123,17 +118,29 @@ def _parse_forecast_row( open_upper_bound=_parse_open_upper_bound(row), open_lower_bound=_parse_open_lower_bound(row), zero_point=_parse_zero_point(row), + inbound_outcome_count=_parse_inbound_outcome_count(row), created_at=pd.to_datetime(row["created_at"]), project=row["project_title"], + unscored_resolution_reason=unscored_resolution_reason, ) question_cache[question_id] = question if username in user_cache: user = user_cache[username] else: + is_bot = _parse_optional_bool(row, "is_bot") + if is_bot is None: + actual_user_type = user_type + else: + actual_user_type = UserType.BOT if is_bot else UserType.PRO + user = User( name=username, - type=user_type, + type=actual_user_type, aggregated_users=[], + is_primary_bot=_parse_optional_bool(row, "is_primary_bot"), + exclude_from_aggregations=_parse_optional_bool( + row, "exclude_from_aggregations" + ), ) user_cache[username] = user @@ -181,7 +188,7 @@ def _parse_forecast(forecast_row: dict) -> ForecastType: prediction = eval(probability_yes_per_category) else: prediction = None - elif question_type == "numeric": + elif question_type == "numeric" or question_type == "discrete": continuous_cdf = row["continuous_cdf"] if pd.notnull(continuous_cdf): prediction = eval(continuous_cdf) @@ -205,7 +212,7 @@ def _parse_resolution(forecast_row: dict) -> ResolutionType: q_type = forecast_row["type"] raw_resolution = forecast_row["resolution"] if pd.isnull(raw_resolution): - raise ValueError(f"Some questions are not resolved. Resolution: {raw_resolution}. Row: {forecast_row}") + return None if str(raw_resolution).lower() in [ "annulled", "ambiguous", @@ -219,7 +226,7 @@ def _parse_resolution(forecast_row: dict) -> ResolutionType: raise ValueError(f"Invalid resolution: {raw_resolution}") elif q_type == "multiple_choice": return str(raw_resolution) - elif q_type == "numeric": + elif q_type == "numeric" or q_type == "discrete": if raw_resolution == "above_upper_bound": return 1000000000000000000000000000000000.0 # Make it super obvious this is a fake number that is above upper bount if raw_resolution == "below_lower_bound": @@ -229,6 +236,19 @@ def _parse_resolution(forecast_row: dict) -> ResolutionType: return raw_resolution +def _parse_unscored_resolution_reason(forecast_row: dict) -> str | None: + """Why resolution is non-scoring: annulled/ambiguous from CSV, or blank (often deleted).""" + raw_resolution = forecast_row["resolution"] + if pd.isnull(raw_resolution): + return "blank" + lower_resolution = str(raw_resolution).lower() + if lower_resolution == "annulled": + return "annulled" + if lower_resolution == "ambiguous": + return "ambiguous" + return None + + def _parse_options(forecast_row: dict) -> tuple[str, ...] | None: if forecast_row["type"] == "multiple_choice": options = forecast_row.get("options") @@ -243,7 +263,7 @@ def _parse_options(forecast_row: dict) -> tuple[str, ...] | None: def _parse_upper_bound(forecast_row: dict) -> float | None: - if forecast_row["type"] == "numeric": + if forecast_row["type"] == "numeric" or forecast_row["type"] == "discrete": upper = forecast_row.get("range_max") if upper is not None and pd.notnull(upper) and upper != "": return float(upper) @@ -252,7 +272,7 @@ def _parse_upper_bound(forecast_row: dict) -> float | None: def _parse_lower_bound(forecast_row: dict) -> float | None: - if forecast_row["type"] == "numeric": + if forecast_row["type"] == "numeric" or forecast_row["type"] == "discrete": lower = forecast_row.get("range_min") if lower is not None and pd.notnull(lower) and lower != "": return float(lower) @@ -260,8 +280,60 @@ def _parse_lower_bound(forecast_row: dict) -> float | None: return None +def _parse_optional_datetime(row: dict, field_name: str) -> pd.Timestamp | None: + value = row.get(field_name) + if value is None or pd.isnull(value): + return None + return pd.to_datetime(value) + + +def _parse_optional_bool(row: dict, field_name: str) -> bool | None: + if field_name not in row: + return None + value = row[field_name] + if value is None or (isinstance(value, float) and pd.isnull(value)): + return None + if isinstance(value, (bool, np.bool_)): + return bool(value) + if isinstance(value, str): + if value == "True": + return True + if value == "False": + return False + return None + return bool(value) + + +def _resolve_spot_scoring_time(row: dict) -> pd.Timestamp: + """Match Metaculus Question.get_spot_scoring_time().""" + spot_scoring_time = _parse_optional_datetime(row, "spot_scoring_time") + if spot_scoring_time is not None: + return spot_scoring_time + + cp_reveal_time = _parse_optional_datetime(row, "cp_reveal_time") + open_time = _parse_optional_datetime(row, "open_time") + if ( + cp_reveal_time is not None + and open_time is not None + and cp_reveal_time > open_time + ): + return cp_reveal_time + + actual_close_time = _parse_optional_datetime(row, "actual_close_time") + if actual_close_time is not None: + return actual_close_time + + scheduled_close_time = _parse_optional_datetime(row, "scheduled_close_time") + if scheduled_close_time is not None: + return scheduled_close_time + + raise ValueError( + f"Could not resolve spot_scoring_time for question_id={row.get('question_id')}" + ) + + def _parse_zero_point(forecast_row: dict) -> float | None: - if forecast_row["type"] == "numeric": + if forecast_row["type"] == "numeric" or forecast_row["type"] == "discrete": zero_point = forecast_row.get("zero_point") if pd.isna(zero_point): return None @@ -270,9 +342,19 @@ def _parse_zero_point(forecast_row: dict) -> float | None: raise ValueError(f"Invalid zero point: {zero_point}") return None +def _parse_inbound_outcome_count(forecast_row: dict) -> int | None: + if forecast_row["type"] == "discrete": + inbound_outcome_count = forecast_row.get("inbound_outcome_count") + if pd.isna(inbound_outcome_count): + return None + elif inbound_outcome_count is not None and pd.notnull(inbound_outcome_count) and inbound_outcome_count != "": + return int(inbound_outcome_count) + raise ValueError(f"Invalid inbound_outcome_count: {inbound_outcome_count}") + return None + def _parse_open_upper_bound(forecast_row: dict) -> bool | None: - if forecast_row["type"] == "numeric": + if forecast_row["type"] == "numeric" or forecast_row["type"] == "discrete": open_upper = forecast_row.get("open_upper_bound") if open_upper is not None and pd.notnull(open_upper) and open_upper != "": return _parse_truth_value(open_upper) @@ -281,7 +363,7 @@ def _parse_open_upper_bound(forecast_row: dict) -> bool | None: def _parse_open_lower_bound(forecast_row: dict) -> bool | None: - if forecast_row["type"] == "numeric": + if forecast_row["type"] == "numeric" or forecast_row["type"] == "discrete": open_lower = forecast_row.get("open_lower_bound") if open_lower is not None and pd.notnull(open_lower) and open_lower != "": return _parse_truth_value(open_lower) diff --git a/aib_analysis/main_logic/process_tournament.py b/aib_analysis/main_logic/process_tournament.py index 3303092..6919688 100644 --- a/aib_analysis/main_logic/process_tournament.py +++ b/aib_analysis/main_logic/process_tournament.py @@ -1,7 +1,7 @@ import copy import logging -import os from datetime import timedelta, timezone +from pathlib import Path from typing import Literal import numpy as np @@ -50,6 +50,10 @@ def combine_tournaments( tournament_2: SimulatedTournament, use_tourn_1_weights: bool, ) -> SimulatedTournament: + """Merge two tournaments into one by pairing questions (hash or force-match) and keeping forecasts from both sides. + + Annulled/ambiguous questions are skipped and never paired, since they cannot be scored. + """ logger.info(f"Combining tournaments {tournament_1.name} and {tournament_2.name}") if ( @@ -65,7 +69,11 @@ def combine_tournaments( matching_questions: dict[str, list[Question]] = {} for question_1 in tournament_1.questions: + if question_1.is_annulled_or_ambiguous: + continue for question_2 in tournament_2.questions: + if question_2.is_annulled_or_ambiguous: + continue hash_1 = question_1.get_hash_for_tournament_matching() hash_2 = question_2.get_hash_for_tournament_matching() dictionary_key = f"{hash_1}_{hash_2}" @@ -233,14 +241,27 @@ def smart_remove_questions_from_tournament( questions_to_exclude: list[Question], use_tournament_matching_hash: bool = True, ) -> SimulatedTournament: + """Drop questions from `tournament` that match any in `questions_to_exclude` (by matching hash or force-match). + + Typical use: build a bot-only qualification set by removing questions that also appear on the pro tournament. + """ if not use_tournament_matching_hash: raise NotImplementedError("Not implemented") + # Annulled/ambiguous questions are not scored and should not drive removals + active_questions_to_exclude = [ + question + for question in questions_to_exclude + if not question.is_annulled_or_ambiguous + ] + if len(active_questions_to_exclude) > 0: + logger.warning(f"Not removing {len(questions_to_exclude) - len(active_questions_to_exclude)} annulled/ambiguous questions from tournament {tournament.name} even though its in the exclude list") + final_questions_to_include = [] all_matches_in_current_tournament: list[list[Question]] = [] for current_question in tournament.questions: matches_with_current_question: list[Question] = [] - for question_to_exclude in questions_to_exclude: + for question_to_exclude in active_questions_to_exclude: exclude_hash = question_to_exclude.get_hash_for_tournament_matching() current_hash = current_question.get_hash_for_tournament_matching() if current_hash == exclude_hash: @@ -261,9 +282,16 @@ def smart_remove_questions_from_tournament( initial_questions_count = len(tournament.questions) num_questions_removed = initial_questions_count - len(final_questions_to_include) - if num_questions_removed != len(questions_to_exclude): + if num_questions_removed != len(active_questions_to_exclude): + skipped_annulled = len(questions_to_exclude) - len(active_questions_to_exclude) + annulled_note = ( + f" (skipped {skipped_annulled} annulled/ambiguous from exclude list)" + if skipped_annulled + else "" + ) logger.warning( - f"{len(questions_to_exclude)} questions were supposed to be removed from tournament. Instead, {num_questions_removed} removals were made." + f"{len(active_questions_to_exclude)} questions were supposed to be removed from tournament. " + f"Instead, {num_questions_removed} removals were made.{annulled_note}" ) for matches_with_current_question in all_matches_in_current_tournament: @@ -279,12 +307,12 @@ def smart_remove_questions_from_tournament( ] if len(filtered_forecasts) == 0: raise ValueError( - f"No forecasts left after removing {len(questions_to_exclude)} questions from {tournament.name}" + f"No forecasts left after removing {len(active_questions_to_exclude)} questions from {tournament.name}" ) return SimulatedTournament( forecasts=filtered_forecasts, - name=f"{tournament.name} ({len(questions_to_exclude)} Questions removed)", + name=f"{tournament.name} ({len(active_questions_to_exclude)} Questions removed)", ) @@ -477,11 +505,12 @@ def save_tournament( else: count_to_use = counter_override non_json_name = file_name.replace(".json", "") - save_path = f"{folder}{count_to_use}_{non_json_name}" + folder_path = Path(folder) + save_stem = folder_path / f"{count_to_use}_{non_json_name}" logger.info(f"Saving tournament {count_to_use} of {non_json_name}") - os.makedirs(folder, exist_ok=True) + folder_path.mkdir(parents=True, exist_ok=True) - _save_specific_tournament_to_file(tournament_to_save, f"{save_path}.json") + _save_specific_tournament_to_file(tournament_to_save, f"{save_stem}.json") if divide_into_types: binary_combined_tournament = constrain_question_types( @@ -490,7 +519,7 @@ def save_tournament( if binary_combined_tournament is not None: _save_specific_tournament_to_file( - binary_combined_tournament, f"{save_path}__binary.json" + binary_combined_tournament, f"{save_stem}__binary.json" ) multiple_choice_combined_tournament = constrain_question_types( @@ -500,7 +529,7 @@ def save_tournament( if multiple_choice_combined_tournament is not None: _save_specific_tournament_to_file( multiple_choice_combined_tournament, - f"{save_path}__multiple_choice.json", + f"{save_stem}__multiple_choice.json", ) numeric_combined_tournament = constrain_question_types( @@ -508,7 +537,7 @@ def save_tournament( ) if numeric_combined_tournament is not None: _save_specific_tournament_to_file( - numeric_combined_tournament, f"{save_path}__numeric.json" + numeric_combined_tournament, f"{save_stem}__numeric.json" ) diff --git a/aib_analysis/main_logic/visualize_tournament.py b/aib_analysis/main_logic/visualize_tournament.py index 1b22c97..cde8f4c 100644 --- a/aib_analysis/main_logic/visualize_tournament.py +++ b/aib_analysis/main_logic/visualize_tournament.py @@ -5,9 +5,10 @@ import plotly.graph_objects as go import streamlit as st -from aib_analysis.data_structures.custom_types import QuestionType +from aib_analysis.data_structures.custom_types import QuestionType, UserType from aib_analysis.data_structures.data_models import ( Leaderboard, + LeaderboardEntry, Question, Score, ScoreType, @@ -21,9 +22,6 @@ find_question_titles_unique_to_first_tournament, get_leaderboard, ) -from aib_analysis.data_structures.data_models import ( - LeaderboardEntry, -) from aib_analysis.math.stats import ( MeanHypothesisCalculator, HypothesisTest, @@ -64,13 +62,15 @@ def display_tournament_and_variations( def display_individual_tournament(tournament: SimulatedTournament, name: str): st.subheader(f"{name}") + hidden_user_types = infer_leaderboard_display_exclusions(name) + # Display tournament statistics with st.expander(f"{name} Spot Peer Leaderboard"): leaderboard = get_leaderboard(tournament, ScoreType.SPOT_PEER) - display_leaderboard(leaderboard) + display_leaderboard(leaderboard, hidden_user_types=hidden_user_types) with st.expander(f"{name} Spot Baseline Leaderboard"): leaderboard = get_leaderboard(tournament, ScoreType.SPOT_BASELINE) - display_leaderboard(leaderboard) + display_leaderboard(leaderboard, hidden_user_types=hidden_user_types) with st.expander(f"{name} Stats"): display_tournament_stats(tournament) with st.expander(f"{name} Forecasts"): @@ -329,13 +329,76 @@ def display_scores(scores: list[Score]): st.dataframe(df, use_container_width=True) -def display_leaderboard(leaderboard: Leaderboard): +def infer_leaderboard_display_exclusions(tournament_name: str) -> set[UserType]: + """Users kept in scoring/peer pools but hidden from LB charts/tables. + + Bot tournaments may include humans so peer GMs match Metaculus; pro tournaments + may include coherence/key-factor bots. Mixed comparison views show everyone. + """ + normalized = ( + tournament_name.lower().replace("|", " ").replace("_", " ").replace("-", " ") + ) + if " vs " in f" {normalized} " or "with bot" in normalized: + return set() + if "pro" in normalized and "bot" not in normalized: + return {UserType.BOT} + if "bot" in normalized: + return {UserType.PRO} + return set() + + +def filter_leaderboard_for_display( + leaderboard: Leaderboard, hidden_user_types: set[UserType] +) -> tuple[Leaderboard, list[LeaderboardEntry]]: + if not hidden_user_types: + return leaderboard, [] + visible_entries = [ + entry + for entry in leaderboard.entries + if entry.user.type not in hidden_user_types + ] + hidden_entries = [ + entry + for entry in leaderboard.entries + if entry.user.type in hidden_user_types + ] + return ( + Leaderboard(entries=visible_entries, type=leaderboard.type), + hidden_entries, + ) + + +def _note_hidden_leaderboard_users(hidden_entries: list[LeaderboardEntry]) -> None: + if not hidden_entries: + return + by_type: dict[str, list[str]] = {} + for entry in hidden_entries: + by_type.setdefault(entry.user.type.value, []).append(entry.user.name) + parts = [] + for user_type, names in sorted(by_type.items()): + sample = ", ".join(sorted(names)[:8]) + extra = f" (+{len(names) - 8} more)" if len(names) > 8 else "" + parts.append(f"{len(names)} {user_type} ({sample}{extra})") + st.info( + "Excluded from this leaderboard chart/table (still included in scoring / peer " + f"baselines where applicable): {'; '.join(parts)}." + ) + + +def display_leaderboard( + leaderboard: Leaderboard, + hidden_user_types: set[UserType] | None = None, +) -> None: confidence_level = 0.95 - _display_average_scores_plot(leaderboard, confidence_level) - _display_leaderboard_table(leaderboard, confidence_level) + display_board, hidden_entries = filter_leaderboard_for_display( + leaderboard, hidden_user_types or set() + ) + _note_hidden_leaderboard_users(hidden_entries) + _display_average_scores_plot(display_board, confidence_level) + _display_leaderboard_table(display_board, confidence_level) _display_score_histogram_by_user( - leaderboard.all_scores, - title="All Users Scores Histogram (overlay, not stacked)", + display_board.all_scores, + title="Displayed Users Scores Histogram (overlay, not stacked)", ) @@ -648,46 +711,76 @@ def display_aggregate_comparison(team_comparison_tourns: list[SimulatedTournamen leaderboard = get_leaderboard(tournament, ScoreType.SPOT_PEER) bot_entry = [entry for entry in leaderboard.entries if entry.user == bot_team_user][0] entries_to_graph.append(bot_entry) - + st.subheader("Aggregate comparison") with st.expander("Aggregate comparison"): - entries_to_graph = sorted(entries_to_graph, key=lambda x: len(x.user.aggregated_users)) - - scores = [entry.average_score for entry in entries_to_graph] - confidence_intervals = [entry.get_confidence_interval(confidence_level=0.95) for entry in entries_to_graph] - lower_bounds = [ci.lower_bound for ci in confidence_intervals] - upper_bounds = [ci.upper_bound for ci in confidence_intervals] - - error_y_minus = [score - lower for score, lower in zip(scores, lower_bounds)] - error_y_plus = [upper - score for score, upper in zip(scores, upper_bounds)] - - team_sizes = [len(entry.user.aggregated_users) for entry in entries_to_graph] - + entries_to_graph = sorted( + entries_to_graph, key=lambda entry: len(entry.user.aggregated_users) + ) + + scores: list[float] = [] + team_sizes: list[int] = [] + error_y_minus: list[float | None] = [] + error_y_plus: list[float | None] = [] + confidence_interval_warnings: list[str] = [] + + for entry in entries_to_graph: + team_size = len(entry.user.aggregated_users) + score = entry.average_score + team_sizes.append(team_size) + scores.append(score) + try: + confidence_interval = entry.get_confidence_interval( + confidence_level=0.95 + ) + error_y_minus.append(score - confidence_interval.lower_bound) + error_y_plus.append(confidence_interval.upper_bound - score) + except ValueError as error: + error_y_minus.append(None) + error_y_plus.append(None) + confidence_interval_warnings.append( + f"team size {team_size} ({entry.question_count} scores): {error}" + ) + + if confidence_interval_warnings: + st.warning( + "T-based confidence intervals omitted for some points " + "(normality assumption failed with n < 30):\n\n" + + "\n".join( + f"- {warning}" for warning in confidence_interval_warnings + ) + ) + fig = go.Figure() - fig.add_trace(go.Scatter( - x=team_sizes, - y=scores, - mode='markers+lines', - name='Bot Team Score', - marker={'size': 10}, - error_y={ - 'type': 'data', - 'symmetric': False, - 'array': error_y_plus, - 'arrayminus': error_y_minus, - 'visible': True - } - )) - + fig.add_trace( + go.Scatter( + x=team_sizes, + y=scores, + mode="markers+lines", + name="Bot Team Score", + marker={"size": 10}, + error_y={ + "type": "data", + "symmetric": False, + "array": error_y_plus, + "arrayminus": error_y_minus, + "visible": True, + }, + ) + ) + fig.update_layout( - title='Bot Team Performance vs Team Size', - xaxis_title='Team Size', - yaxis_title='Average Score', - hovermode='closest', - showlegend=True + title="Bot Team Performance vs Team Size", + xaxis_title="Team Size", + yaxis_title="Average Score", + hovermode="closest", + showlegend=True, ) - + st.plotly_chart(fig, use_container_width=True) for entry in entries_to_graph: - st.write(f"- Bot Team Size: {len(entry.user.aggregated_users)} | Score: {entry.average_score:.3f}") + st.write( + f"- Bot Team Size: {len(entry.user.aggregated_users)} | " + f"Score: {entry.average_score:.3f}" + ) diff --git a/aib_analysis/math/aggregate.py b/aib_analysis/math/aggregate.py index be7b2d1..9007102 100644 --- a/aib_analysis/math/aggregate.py +++ b/aib_analysis/math/aggregate.py @@ -8,6 +8,7 @@ from aib_analysis.data_structures.custom_types import ( BinaryForecastType, + DiscreteForecastType, MCForecastType, NumericForecastType, QuestionType, @@ -139,9 +140,11 @@ def aggregate_forecasts( mc_predictions = [forecast.prediction for forecast in forecasts] mc_predictions = typeguard.check_type(mc_predictions, list[MCForecastType]) aggregated_forecast = _aggregate_mc_forecasts(mc_predictions) - elif question_type == QuestionType.NUMERIC: + elif question_type == QuestionType.NUMERIC or question_type == QuestionType.DISCRETE: numeric_predictions = [forecast.prediction for forecast in forecasts] - numeric_predictions = typeguard.check_type(numeric_predictions, list[NumericForecastType]) + numeric_predictions = typeguard.check_type( + numeric_predictions, list[NumericForecastType | DiscreteForecastType] + ) aggregated_forecast = _aggregate_numeric_forecasts(numeric_predictions) else: raise ValueError(f"Unknown question type: {question_type}") diff --git a/aib_analysis/math/scoring.py b/aib_analysis/math/scoring.py index 53884dc..377d47f 100644 --- a/aib_analysis/math/scoring.py +++ b/aib_analysis/math/scoring.py @@ -19,7 +19,8 @@ def calculate_peer_score( range_min: float | None = None, range_max: float | None = None, question_weight: float = 1.0, - q_type: Literal["binary", "multiple_choice", "numeric"] | None = None, + q_type: Literal["binary", "multiple_choice", "numeric", "discrete"] | None = None, + zero_point: float | None = None, ) -> float: if len(forecast_for_other_users) == 0: return 0.0 @@ -27,19 +28,25 @@ def calculate_peer_score( question_type = _determine_question_type(q_type, resolution) resolution = _normalize_resolution(question_type, resolution, range_min, range_max) forecast_for_resolution = _determine_probability_for_resolution( - question_type, forecast, resolution, options, range_min, range_max + question_type, forecast, resolution, options, range_min, range_max, zero_point ) other_user_forecasts = [ _determine_probability_for_resolution( - question_type, forecast, resolution, options, range_min, range_max + question_type, f, resolution, options, range_min, range_max, zero_point ) - for forecast in forecast_for_other_users + for f in forecast_for_other_users ] geometric_mean = gmean(other_user_forecasts) peer_score = np.log(forecast_for_resolution / geometric_mean) - if question_type == QuestionType.NUMERIC: + + n_forecasters = len(other_user_forecasts) + if n_forecasters > 1: + peer_score *= (n_forecasters / (n_forecasters - 1)) + + if question_type == QuestionType.NUMERIC or question_type == QuestionType.DISCRETE: peer_score /= 2 + return peer_score * question_weight * 100 @@ -52,7 +59,8 @@ def calculate_baseline_score( question_weight: float = 1.0, open_upper_bound: bool | None = False, open_lower_bound: bool | None = False, - q_type: Literal["binary", "multiple_choice", "numeric"] | None = None, + q_type: Literal["binary", "multiple_choice", "numeric", "discrete"] | None = None, + zero_point: float | None = None, ) -> float: """ Question type can be infered from resolution type @@ -61,11 +69,12 @@ def calculate_baseline_score( question_type = _determine_question_type(q_type, resolution) resolution = _normalize_resolution(question_type, resolution, range_min, range_max) prob_for_resolution = _determine_probability_for_resolution( - question_type, forecast, resolution, options, range_min, range_max + question_type, forecast, resolution, options, range_min, range_max, zero_point ) baseline_prob = _determine_baseline( question_type, resolution, + forecast, options, range_min, range_max, @@ -88,6 +97,7 @@ def calculate_baseline_score( def _determine_baseline( question_type: QuestionType, resolution: ResolutionType, + forecast: ForecastType, options: list[str] | None = None, range_min: float | None = None, range_max: float | None = None, @@ -101,17 +111,17 @@ def _determine_baseline( if options is None: raise ValueError("Options are required for multiple choice questions") baseline_prob = 1 / len(options) - elif question_type == QuestionType.NUMERIC: + elif question_type == QuestionType.NUMERIC or question_type == QuestionType.DISCRETE: if open_upper_bound is None or open_lower_bound is None: raise ValueError( - "Open upper bound and lower bound are required for numeric questions" + f"Open upper bound and lower bound are required for {question_type.value} questions" ) if range_min is None or range_max is None: raise ValueError( - "Range min and range max are required for numeric questions" + f"Range min and range max are required for {question_type.value} questions" ) if not isinstance(resolution, float): - raise ValueError("Resolution must be a float for numeric questions") + raise ValueError(f"Resolution must be a float for {question_type.value} questions") resolved_outside_bounds = False @@ -124,9 +134,10 @@ def _determine_baseline( baseline_prob = 0.05 else: open_bound_count = bool(open_upper_bound) + bool(open_lower_bound) + pmf_inner_bins = len(forecast) - 1 # len(forecast) is len(cdf). For numeric PMF has 202 bins, 2 of which represent the bounds. CDF is 201. So 201 -1 = 200 is the internal bins baseline_prob = ( 1 - 0.05 * open_bound_count - ) / 200 # PMF has 202 bins, 2 of which represent the bounds. So 200 is the internal bins + ) / pmf_inner_bins else: raise ValueError("Unknown question type") assert ( @@ -142,6 +153,7 @@ def _determine_probability_for_resolution( options: list[str] | None = None, range_min: float | None = None, range_max: float | None = None, + zero_point: float | None = None, ) -> float: """ Returns a 0 to 1 probability for the resolution @@ -167,7 +179,7 @@ def _determine_probability_for_resolution( f"Error encountered for question of type {q_type} with resolution {resolution} and forecast {forecast}: {e}" ) - if not q_type == QuestionType.NUMERIC and any(p <= 0 or p >= 1 for p in forecast): + if not (q_type == QuestionType.NUMERIC or q_type == QuestionType.DISCRETE) and any(p < 0 or p > 1 for p in forecast): raise ValueError("Forecast contains probabilities outside of 0 to 1 range") if q_type == QuestionType.BINARY: @@ -180,16 +192,16 @@ def _determine_probability_for_resolution( prob_for_resolution = _multiple_choice_resolution_prob( forecast, resolution, options ) - elif q_type == QuestionType.NUMERIC: + elif q_type == QuestionType.NUMERIC or q_type == QuestionType.DISCRETE: if range_min is None or range_max is None: raise ValueError( - "Range min and range max are required for numeric questions" + f"Range min and range max are required for {q_type.value} questions" ) assert isinstance( resolution, float ), f"Resolution is {resolution} which is not a float" prob_for_resolution = _numeric_resolution_prob( - forecast, resolution, range_min, range_max + forecast, resolution, range_min, range_max, zero_point ) else: raise ValueError(f"Unknown question type: {q_type}") @@ -231,11 +243,8 @@ def _multiple_choice_resolution_prob( def _numeric_resolution_prob( - forecast: list[float], resolution: float, range_min: float, range_max: float + forecast: list[float], resolution: float, range_min: float, range_max: float, zero_point: float | None = None ) -> float: - if len(forecast) != 201: - raise ValueError("CDF should have 201 bins") - previous_prob = 0 for current_prob in forecast: if current_prob < previous_prob: @@ -246,7 +255,7 @@ def _numeric_resolution_prob( pmf = cdf_to_pmf(cdf) resolution_bin_idx = _resolution_value_to_pmf_index( - pmf, resolution, range_min, range_max + pmf, resolution, range_min, range_max, zero_point ) prob_for_resolution = pmf[resolution_bin_idx] @@ -267,28 +276,27 @@ def _determine_divisor_for_baseline_score( if options is None: raise ValueError("Options are required for multiple choice questions") return np.log(len(options)) - elif question_type == QuestionType.NUMERIC: + elif question_type == QuestionType.NUMERIC or question_type == QuestionType.DISCRETE: return 2 else: raise ValueError("Unknown question type") def _resolution_value_to_pmf_index( - pmf: list[float], resolution: float, range_min: float, range_max: float + pmf: list[float], resolution: float, range_min: float, range_max: float, zero_point: float | None = None ) -> int: """ PMF explanation: - - 200 bins for the internal range + - `outcome_count` bins for the internal range - 1 bin for the 'above upper bound' - 1 bin for the 'below lower bound' - - 202 total bins + - `outcome_count + 2` total bins """ - if len(pmf) != 202: - raise ValueError(f"PMF should have 202 bins, but has {len(pmf)}") + outcome_count = len(pmf) - 2 position_in_range = _resolution_value_to_position_in_numeric_range( - resolution, range_min, range_max + resolution, range_min, range_max, zero_point ) - resolution_bin_idx = _position_in_range_to_bucket_index(position_in_range) + resolution_bin_idx = _position_in_range_to_bucket_index(position_in_range, outcome_count) if resolution_bin_idx >= len(pmf) or resolution_bin_idx < 0: raise ValueError( f"Invalid resolution bin index: {resolution_bin_idx}. Resolution: {resolution}, Range min: {range_min}, Range max: {range_max}" @@ -299,9 +307,8 @@ def _resolution_value_to_pmf_index( return resolution_bin_idx def _position_in_range_to_bucket_index( - position_in_range: float + position_in_range: float, outcome_count: int ) -> int: - outcome_count = 200 if position_in_range < 0: return 0 if position_in_range > 1: @@ -441,7 +448,6 @@ def _determine_question_type( def cdf_to_pmf(cdf: list[float]) -> list[float]: - assert len(cdf) == 201, f"There should be 201 bins, but there are {len(cdf)}" lower_bound_prob = cdf[0] upper_bound_prob = 1 - cdf[-1] pmf = ( @@ -449,17 +455,16 @@ def cdf_to_pmf(cdf: list[float]) -> list[float]: + [cdf[i] - cdf[i - 1] for i in range(1, len(cdf))] + [upper_bound_prob] ) - assert len(pmf) == 202, f"There should be 202 bins, but there are {len(pmf)}" + assert len(pmf) == len(cdf) + 1, f"There should be {len(cdf) + 1} bins, but there are {len(pmf)}" return pmf def pmf_to_cdf(pmf: list[float]) -> list[float]: - assert len(pmf) == 202, f"There should be 202 bins, but there are {len(pmf)}" cdf = [] total = 0.0 for p in pmf: total += p cdf.append(total) - assert len(cdf) == 201, f"There should be 201 bins, but there are {len(cdf)}" + assert len(cdf) == len(pmf) - 1, f"There should be {len(pmf) - 1} bins, but there are {len(cdf)}" return cdf diff --git a/aib_analysis/print_leaderboards.py b/aib_analysis/print_leaderboards.py new file mode 100644 index 0000000..fe98c8f --- /dev/null +++ b/aib_analysis/print_leaderboards.py @@ -0,0 +1,41 @@ +import json +import os +import sys + +current_dir = os.path.dirname(os.path.abspath(__file__)) +top_level_dir = os.path.abspath(os.path.join(current_dir, "../")) +sys.path.append(top_level_dir) + +from aib_analysis.main_logic.process_tournament import get_leaderboard +from aib_analysis.data_structures.simulated_tournament import SimulatedTournament +from aib_analysis.data_structures.custom_types import ScoreType + +def print_leaderboard(file_path): + print(f"Leaderboard for {file_path}:") + with open(file_path, "r") as f: + data = json.load(f) + + tournament = SimulatedTournament(**data) + leaderboard = get_leaderboard(tournament, ScoreType.SPOT_PEER) + + sorted_entries = leaderboard.entries_via_sum_of_scores() + + for i, entry in enumerate(sorted_entries[:20]): + user_name = entry.user.name + total_score = entry.sum_of_scores + questions = entry.question_count + print(f"{i+1}. {user_name}: {total_score:.3f} (Questions: {questions})") + print() + +if __name__ == "__main__": + folder = "local/spring_2026_simulations_teams_comparison" + try: + print_leaderboard(os.path.join(folder, "5_pro_tournament__no_teams.json")) + except Exception as e: + print(f"Could not load pro: {e}") + + try: + print_leaderboard(os.path.join(folder, "2_bot_tournament.json")) + except Exception as e: + print(f"Could not load bot: {e}") + diff --git a/aib_analysis/run_seasonal_analysis.py b/aib_analysis/run_seasonal_analysis.py deleted file mode 100644 index f345806..0000000 --- a/aib_analysis/run_seasonal_analysis.py +++ /dev/null @@ -1,255 +0,0 @@ -import logging -import os -import random -import sys - -current_dir = os.path.dirname(os.path.abspath(__file__)) -top_level_dir = os.path.abspath(os.path.join(current_dir, "../")) -sys.path.append(top_level_dir) - -from aib_analysis.data_structures.data_models import Forecast, UserType -from aib_analysis.data_structures.simulated_tournament import ( - SimulatedTournament, -) -from aib_analysis.main_logic.load_tournament import load_tournament -from aib_analysis.main_logic.process_tournament import ( - combine_tournaments, - create_team_tournament, - get_best_forecasters_from_tournament, - save_tournament, - smart_remove_questions_from_tournament, -) -from conftest import initialize_logging - -logger = logging.getLogger(__name__) - - -analysis_section_counter: int = 0 - - -def next_count() -> int: - global analysis_section_counter - analysis_section_counter += 1 - return analysis_section_counter - - -def main( - cp_path: str, - bot_path: str, - output_folder: str, -): - initialize_logging() - random_seed = 42 - main_cp_size = 15 - main_team_size = 10 - random.seed(random_seed) - - # ----------------------- Base CP and Bot Tournament ----------------------- - cp_tournament = grab_tournament_data(cp_path, UserType.CP, "CP Tournament") - save_tournament( - cp_tournament, - "cp_tournament.json", - folder=output_folder, - counter_override=next_count(), - ) - - bot_tournament_full = grab_tournament_data( - bot_path, UserType.BOT, "Bot Tournament Full" - ) - bot_tournament = SimulatedTournament( - name="Bot Tournament (Only spot forecasts)", - forecasts=bot_tournament_full.spot_forecasts, - ) - save_tournament( - bot_tournament, - "bot_tournament.json", - divide_into_types=True, - folder=output_folder, - counter_override=next_count(), - ) - - # ----------------------- Train Test Split ----------------------- - test_set_fraction = 0.67 # Give extra room so that the cp forecaster filter works - test_set_size = int(len(cp_tournament.questions) * test_set_fraction) - test_set_save_number = next_count() - - full_cp_test_set_questions = random.sample(cp_tournament.questions, test_set_size) - full_cp_test_set_forecasts = [ - forecast - for forecast in cp_tournament.forecasts - if forecast.question in full_cp_test_set_questions - ] - full_cp_test_set_tournament = SimulatedTournament( - name="Full CP Test Set Tournament", - forecasts=full_cp_test_set_forecasts, - ) - save_tournament( - full_cp_test_set_tournament, - "full_cp_test_set_tournament.json", - folder=output_folder, - counter_override=test_set_save_number, - ) - - quality_cp_test_set_forecasts = filter_by_cp_size( - full_cp_test_set_tournament, main_cp_size - ) - quality_cp_test_set_tournament = SimulatedTournament( - name="Quality CP Test Set Tournament", - forecasts=quality_cp_test_set_forecasts, - ) - save_tournament( - quality_cp_test_set_tournament, - "quality_cp_test_set_tournament.json", - folder=output_folder, - counter_override=test_set_save_number, - ) - - # ----------------------- Bot Qualification Tournament ----------------------- - - bot_team_qualification_tournament = smart_remove_questions_from_tournament( - tournament=bot_tournament, - questions_to_exclude=full_cp_test_set_tournament.questions, - ) - save_tournament( - bot_team_qualification_tournament, - "bot_team_qualification_tournament.json", - divide_into_types=True, - folder=output_folder, - counter_override=next_count(), - ) - - # ------------------------- CP vs Bot team Tournaments ------------------------- - team_comparison_counter = next_count() - use_bot_tourn_weights = True - for bot_team_size in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 30, 50, 100]: - if bot_team_size > len(bot_tournament.users): - continue - cp_team = quality_cp_test_set_tournament.users - bot_team_for_cp_comparison = get_best_forecasters_from_tournament( - bot_team_qualification_tournament, bot_team_size - ) - cp_v_bot_tournament__teams = create_team_tournament( - tournament_1=quality_cp_test_set_tournament, - tournament_2=bot_tournament, - team_1=cp_team, - team_2=bot_team_for_cp_comparison, - aggregate_name_1="CP Team", - aggregate_name_2="Bot Team", - use_tourn_1_weights=not use_bot_tourn_weights, - ) - save_tournament( - cp_v_bot_tournament__teams, - f"cp_vs_bot_tournament__teams_size_{bot_team_size}.json", - divide_into_types=False, - folder=output_folder, - counter_override=team_comparison_counter, - ) - if bot_team_size == main_team_size: - save_tournament( - cp_v_bot_tournament__teams, - f"cp_vs_bot_tournament__teams_size_{main_team_size}.json", - divide_into_types=True, - folder=output_folder, - counter_override=next_count(), - ) - - # ------------------------- Other Tournaments ------------------------- - process_metac_bot_tournament(bot_tournament, output_folder) - process_bot_maker_only_tournament(bot_tournament, output_folder) - process_cp_with_bot_tournament(cp_tournament, bot_tournament, output_folder, main_cp_size) - - -def process_metac_bot_tournament( - bot_tournament: SimulatedTournament, output_folder: str -) -> None: - metac_bot_users = [user for user in bot_tournament.users if user.is_metac_bot] - metac_bot_forecasts = [ - forecast - for user in metac_bot_users - for forecast in bot_tournament.user_to_spot_forecasts(user.name) - ] - - metac_bot_tournament = SimulatedTournament( - name="Metac Bot Tournament", - forecasts=metac_bot_forecasts, - ) - save_tournament( - metac_bot_tournament, - "metac_bot_tournament.json", - folder=output_folder, - counter_override=next_count(), - ) - - -def process_bot_maker_only_tournament( - bot_tournament: SimulatedTournament, output_folder: str -) -> None: - # Display regular participants - regular_bot_maker = [ - user for user in bot_tournament.users if not user.is_metac_bot - ] - regular_bot_maker_forecasts = [ - forecast - for user in regular_bot_maker - for forecast in bot_tournament.user_to_spot_forecasts(user.name) - ] - regular_bot_maker_tournament = SimulatedTournament( - name="Bot Maker Only Tournament", - forecasts=regular_bot_maker_forecasts, - ) - save_tournament( - regular_bot_maker_tournament, - "bot_maker_only_tournament.json", - folder=output_folder, - counter_override=next_count(), - ) - - -def process_cp_with_bot_tournament( - cp_tournament: SimulatedTournament, - bot_tournament: SimulatedTournament, - output_folder: str, - cp_size: int, -) -> None: - all_quality_cp_forecasts = filter_by_cp_size(cp_tournament, cp_size) - all_quality_cp_tournament = SimulatedTournament( - name="All Quality CP Tournament", - forecasts=all_quality_cp_forecasts, - ) - use_bot_tourn_weights = True - cp_with_bot_tourn = combine_tournaments( - all_quality_cp_tournament, - bot_tournament, - use_tourn_1_weights=not use_bot_tourn_weights, - ) - save_tournament( - cp_with_bot_tourn, - "cp_with_bot_tourn__no_teams.json", - divide_into_types=True, - folder=output_folder, - counter_override=next_count(), - ) - - -def filter_by_cp_size(tournament: SimulatedTournament, cp_size: int) -> list[Forecast]: - return [ - forecast - for forecast in tournament.forecasts - if forecast.forecasters_at_time is not None - and forecast.forecasters_at_time >= cp_size - ] - - -def grab_tournament_data( - path: str, user_type: UserType, tournament_name: str -) -> SimulatedTournament: - return load_tournament(path, user_type, tournament_name) - - -if __name__ == "__main__": - - main( - cp_path="local/private_input_data/cp_forecasts_fall.csv", - bot_path="local/private_input_data/bot_forecasts_fall.csv", - output_folder="local/fall_2025_simulations_teams_comparison/", - ) diff --git a/aib_analysis/run_spring_simulations.py b/aib_analysis/run_spring_simulations.py new file mode 100644 index 0000000..c56d094 --- /dev/null +++ b/aib_analysis/run_spring_simulations.py @@ -0,0 +1,292 @@ +import logging +import os +import sys + +current_dir = os.path.dirname(os.path.abspath(__file__)) +top_level_dir = os.path.abspath(os.path.join(current_dir, "../")) +sys.path.append(top_level_dir) + +from aib_analysis.data_structures.data_models import User, UserType +from aib_analysis.data_structures.simulated_tournament import ( + SimulatedTournament, +) +from aib_analysis.main_logic.load_tournament import load_tournament +from aib_analysis.main_logic.process_tournament import ( + combine_tournaments, + create_team_tournament, + create_weighted_q3_spot_forecast_tourn, + get_best_forecasters_from_tournament, + save_tournament, + smart_remove_questions_from_tournament, +) +from conftest import initialize_logging + +logger = logging.getLogger(__name__) + + +def main( + pro_path: str, + bot_path: str, + quarterly_cup_path: str | None, + output_folder: str, +): + initialize_logging() + + local_counter: int = 0 + def next_count() -> int: + nonlocal local_counter + local_counter += 1 + return local_counter + + is_q3 = "q3" in output_folder.lower() + is_q4 = "q4" in output_folder.lower() + + # ----------------------- Pros and Bot Tournaments ----------------------- + pro_tournament = grab_tournament_data(pro_path, UserType.PRO, "Pro Tournament") + save_tournament( + pro_tournament, + "pro_tournament.json", + folder=output_folder, + counter_override=next_count(), + ) + + bot_tournament_full = grab_tournament_data( + bot_path, UserType.BOT, "Bot Tournament Full" + ) + if is_q3: + bot_tournament = create_weighted_q3_spot_forecast_tourn(bot_tournament_full) + else: + bot_tournament = SimulatedTournament( + name="Bot Tournament (Only spot forecasts)", + forecasts=bot_tournament_full.spot_forecasts, + ) + save_tournament( + bot_tournament, + "bot_tournament.json", + divide_into_types=True, + folder=output_folder, + counter_override=next_count(), + ) + + comparison_bot_users = get_comparison_bot_users(bot_tournament) + skip_comparison_team_outputs = False + if len(comparison_bot_users) < 2: + logger.warning( + "Skipping comparison-team outputs: expected 2 or greater cross-tournament control " + f"bots, found {len(comparison_bot_users)}: " + f"{[user.name for user in comparison_bot_users]}" + ) + skip_comparison_team_outputs = True + + bot_tournament_wo_pro_questions = smart_remove_questions_from_tournament( + tournament=bot_tournament, questions_to_exclude=pro_tournament.questions + ) + save_tournament( + bot_tournament_wo_pro_questions, + "bot_tournament_without_pro_questions.json", + divide_into_types=True, + folder=output_folder, + counter_override=next_count(), + ) + + # ------------------------- Subdivisions of Bot Tournament ------------------------- + + metac_bot_users = [user for user in bot_tournament.users if user.is_metac_bot] + metac_bot_forecasts = [ + forecast + for user in metac_bot_users + for forecast in bot_tournament.user_to_spot_forecasts(user.name) + ] + + metac_bot_tournament = SimulatedTournament( + name="Metac Bot Tournament", + forecasts=metac_bot_forecasts, + ) + save_tournament( + metac_bot_tournament, + "metac_bot_tournament.json", + folder=output_folder, + counter_override=next_count(), + ) + + # Display regular participants + regular_participant_users = [ + user for user in bot_tournament.users if not user.is_metac_bot + ] + regular_participant_forecasts = [ + forecast + for user in regular_participant_users + for forecast in bot_tournament.user_to_spot_forecasts(user.name) + ] + regular_participant_tournament = SimulatedTournament( + name="Regular Participant Tournament", + forecasts=regular_participant_forecasts, + ) + save_tournament( + regular_participant_tournament, + "regular_participant_tournament.json", + folder=output_folder, + counter_override=next_count(), + ) + + # ------------------------- Combine Pro and Bot Tournaments ------------------------- + + use_pro_weights = False if is_q3 or is_q4 else True + pro_with_bot_tourn = combine_tournaments( + pro_tournament, bot_tournament, use_tourn_1_weights=use_pro_weights + ) + save_tournament( + pro_with_bot_tourn, + "pro_with_bot_tourn__no_teams.json", + divide_into_types=True, + folder=output_folder, + counter_override=next_count(), + ) + + team_comparison_counter = next_count() + size_10_bot_team = None + for bot_team_size in [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 30, 50, 100]: + if bot_team_size > len(bot_tournament.users): + continue + pro_team = pro_tournament.users + bot_team_for_pro_comparison = get_best_forecasters_from_tournament( + bot_tournament_wo_pro_questions, bot_team_size + ) + pro_v_bot_tournament__teams = create_team_tournament( + tournament_1=pro_tournament, + tournament_2=bot_tournament, + team_1=pro_team, + team_2=bot_team_for_pro_comparison, + aggregate_name_1="Pro Team", + aggregate_name_2="Bot Team", + use_tourn_1_weights=use_pro_weights, + ) + save_tournament( + pro_v_bot_tournament__teams, + f"pro_vs_bot_tournament__teams_size_{bot_team_size}.json", + divide_into_types=False, + folder=output_folder, + counter_override=team_comparison_counter, + ) + if bot_team_size == 10: + save_tournament( + pro_v_bot_tournament__teams, + "pro_vs_bot_tournament__teams_size_10.json", + divide_into_types=True, + folder=output_folder, + counter_override=next_count(), + ) + size_10_bot_team = bot_team_for_pro_comparison + + # ------------------- Control/comparison Bots ------------------- + if not skip_comparison_team_outputs and size_10_bot_team is not None: + number_to_use = 99 + comparison_vs_bot__teams = create_team_tournament( + tournament_1=pro_with_bot_tourn, + tournament_2=pro_with_bot_tourn, + team_1=comparison_bot_users, + team_2=size_10_bot_team, + aggregate_name_1="Comparison Team", + aggregate_name_2="Bot Team", + use_tourn_1_weights=use_pro_weights, + ) + save_tournament( + comparison_vs_bot__teams, + "comparison_vs_bot__teams.json", + folder=output_folder, + divide_into_types=True, + counter_override=number_to_use, + ) + comparison_vs_pros__teams = create_team_tournament( + tournament_1=pro_with_bot_tourn, + tournament_2=pro_with_bot_tourn, + team_1=comparison_bot_users, + team_2=pro_team, + aggregate_name_1="Comparison Team", + aggregate_name_2="Pro Team", + use_tourn_1_weights=use_pro_weights, + ) + save_tournament( + comparison_vs_pros__teams, + "comparison_vs_pro__teams.json", + folder=output_folder, + divide_into_types=True, + counter_override=number_to_use, + ) + + # ------------------- Quarterly Cup ------------------- + if quarterly_cup_path is None: + return + + cup_tournament = grab_tournament_data( + quarterly_cup_path, UserType.BOT, "Quarterly Cup" + ) + save_tournament( + cup_tournament, + "spot_scores_for_quarterly_cup.json", + folder=output_folder, + counter_override=next_count(), + ) + + bot_tournament_wo_cup_questions = smart_remove_questions_from_tournament( + bot_tournament, cup_tournament.questions + ) + save_tournament( + bot_tournament_wo_cup_questions, + "bot_tournament_wo_cup_questions.json", + folder=output_folder, + counter_override=next_count(), + ) + + cup_team_size = 10 + bot_team_for_cup_comparison = get_best_forecasters_from_tournament( + bot_tournament_wo_cup_questions, cup_team_size + ) + cup_vs_bot_teams = create_team_tournament( + tournament_1=cup_tournament, + tournament_2=bot_tournament, + team_1="all", + team_2=bot_team_for_cup_comparison, + aggregate_name_1="Cup Team (All forecasters)", + aggregate_name_2="Bot Team", + use_tourn_1_weights=use_pro_weights, + ) + save_tournament( + cup_vs_bot_teams, + "cup_vs_bot_teams.json", + folder=output_folder, + counter_override=next_count(), + ) + + +def get_comparison_bot_users(bot_tournament: SimulatedTournament) -> list[User]: + # Cross-tournament control pair: same GPT + Claude Metaculus template lineage + # across seasons (account names change; exactly two should match a given CSV). + comparison_bot_names = [ + "metac-gpt-4o+asknews", # Q2 version of gpt-4o + "metac-claude-3-5-sonnet-20240620+asknews", # Q2 version of claude 3.5 sonnet + "metac-gpt-4o", # Q1 version of gpt-4o + "metac-claude-3-5-sonnet-20240620", # Q1 version of claude 3.5 sonnet + "mf-bot-1", # Q3/4 version of gpt-4o + "mf-bot-3", # Q3/4 version of claude 3.5 sonnet + # "metac-claude-3-5-sonnet-latest+asknews", # Wasn't around in Q3 + ] + comparison_bot_users = [ + user for user in bot_tournament.users if user.name in comparison_bot_names + ] + return comparison_bot_users + + +def grab_tournament_data( + path: str, user_type: UserType, tournament_name: str +) -> SimulatedTournament: + return load_tournament(path, user_type, tournament_name) + + +if __name__ == "__main__": + main( + pro_path="local/private_input_data/pro_forecasts_2026_spring.csv", + bot_path="local/private_input_data/bot_forecasts_2026_spring.csv", + quarterly_cup_path=None, + output_folder="local/spring_2026_simulations_teams_comparison/", + ) \ No newline at end of file diff --git a/tests/test_problem_questions.py b/tests/test_problem_questions.py new file mode 100644 index 0000000..cad483a --- /dev/null +++ b/tests/test_problem_questions.py @@ -0,0 +1,64 @@ +from datetime import datetime + +from aib_analysis.data_structures.custom_types import QuestionType +from aib_analysis.data_structures.data_models import Question +from aib_analysis.data_structures.problem_questions import ( + _all_unscored_title_match_message, +) +from aib_analysis.main_logic.load_tournament import _parse_unscored_resolution_reason + + +def _unscored_question(post_id: int, reason: str | None) -> Question: + return Question( + post_id=post_id, + question_id=post_id, + type=QuestionType.BINARY, + question_text="Same title", + resolution=None, + options=None, + range_max=None, + range_min=None, + open_upper_bound=None, + open_lower_bound=None, + weight=1.0, + spot_scoring_time=datetime(2030, 1, 1), + created_at=datetime(2030, 1, 1), + unscored_resolution_reason=reason, + ) + + +def test_parse_unscored_resolution_reason() -> None: + assert _parse_unscored_resolution_reason({"resolution": "annulled"}) == "annulled" + assert _parse_unscored_resolution_reason({"resolution": "ambiguous"}) == "ambiguous" + assert _parse_unscored_resolution_reason({"resolution": float("nan")}) == "blank" + assert _parse_unscored_resolution_reason({"resolution": "yes"}) is None + + +def test_all_unscored_title_match_message_differentiates() -> None: + annulled = [ + _unscored_question(1, "annulled"), + _unscored_question(2, "annulled"), + ] + assert "all are annulled" in _all_unscored_title_match_message(annulled) + + blank = [ + _unscored_question(1, "blank"), + _unscored_question(2, "blank"), + ] + assert "blank resolutions" in _all_unscored_title_match_message(blank) + assert "deleted or unresolved" in _all_unscored_title_match_message(blank) + + mixed = [ + _unscored_question(1, "annulled"), + _unscored_question(2, "blank"), + ] + mixed_message = _all_unscored_title_match_message(mixed) + assert "annulled or deleted" in mixed_message + assert "1 annulled" in mixed_message + assert "1 blank" in mixed_message + + unknown = [ + _unscored_question(1, None), + _unscored_question(2, "annulled"), + ] + assert "annulled or deleted" in _all_unscored_title_match_message(unknown) diff --git a/tests/test_process_tournament.py b/tests/test_process_tournament.py index 6a0f499..0fc523d 100644 --- a/tests/test_process_tournament.py +++ b/tests/test_process_tournament.py @@ -202,7 +202,8 @@ def test_combine_tournaments_pro_plus_bot( pro_users = pro_tournament.users bot_users = bot_tournament.users combined_tournament = combine_tournaments(pro_tournament, bot_tournament, use_tourn_1_weights=True) - assert len(combined_tournament.questions) == 98 + # Annulled/ambiguous hash pairs are skipped when combining (2 fewer than historical 98). + assert len(combined_tournament.questions) == 96 assert len(combined_tournament.forecasts) < len(pro_forecasts) + len( bot_forecasts ) @@ -372,7 +373,8 @@ def test_combine_pro_and_bot_tournament( use_tourn_1_weights=True, ) assert len(aggregate_tournament.users) == 2 - assert len(aggregate_tournament.questions) == 98 + # Annulled/ambiguous hash pairs are skipped when combining (2 fewer than historical 98). + assert len(aggregate_tournament.questions) == 96 leaderboard = get_leaderboard(aggregate_tournament, ScoreType.SPOT_PEER) entries = leaderboard.entries_via_sum_of_scores() assert entries[0].user.name == pro_team_name diff --git a/tests/test_visualize_tournament.py b/tests/test_visualize_tournament.py new file mode 100644 index 0000000..f49d7e0 --- /dev/null +++ b/tests/test_visualize_tournament.py @@ -0,0 +1,56 @@ +from aib_analysis.data_structures.custom_types import ScoreType, UserType +from aib_analysis.data_structures.data_models import Leaderboard, LeaderboardEntry, Score +from aib_analysis.main_logic.visualize_tournament import ( + filter_leaderboard_for_display, + infer_leaderboard_display_exclusions, +) +from tests.mock_data_maker import make_forecast, make_question_binary, make_user + + +def test_infer_leaderboard_display_exclusions() -> None: + assert infer_leaderboard_display_exclusions("Pro Tournament") == {UserType.BOT} + assert infer_leaderboard_display_exclusions("Bot Tournament") == {UserType.PRO} + assert infer_leaderboard_display_exclusions("Bot Tournament | Binary") == { + UserType.PRO + } + assert infer_leaderboard_display_exclusions( + "Pro With Bot Tourn | No Teams" + ) == set() + assert infer_leaderboard_display_exclusions( + "Pro Vs Bot Tournament | Teams Size 10" + ) == set() + + +def test_filter_leaderboard_for_display_hides_user_types() -> None: + question = make_question_binary() + pro_user = make_user(name="pro_user", user_type=UserType.PRO) + bot_user = make_user(name="bot_user", user_type=UserType.BOT) + pro_forecast = make_forecast(user=pro_user, question=question, prediction=[0.7, 0.3]) + bot_forecast = make_forecast(user=bot_user, question=question, prediction=[0.6, 0.4]) + leaderboard = Leaderboard( + type=ScoreType.SPOT_PEER, + entries=[ + LeaderboardEntry( + scores=[ + Score( + score=10.0, + type=ScoreType.SPOT_PEER, + forecast=pro_forecast, + ) + ] + ), + LeaderboardEntry( + scores=[ + Score( + score=20.0, + type=ScoreType.SPOT_PEER, + forecast=bot_forecast, + ) + ] + ), + ], + ) + + visible, hidden = filter_leaderboard_for_display(leaderboard, {UserType.PRO}) + assert [entry.user.name for entry in visible.entries] == ["bot_user"] + assert [entry.user.name for entry in hidden] == ["pro_user"]