diff --git a/CHANGELOG.md b/CHANGELOG.md index 100c90e..df84d94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) ## [Unreleased] ### Added +- A "Results" section in the docs (`docs/results/`): one page per model + ($1K upgrade, response/major gift, lapse, suggested ask, planned giving, + who to mail) written for fundraisers, not data scientists, comparing each + model's picks against a simple rule and against random in plain donor + counts, plus an index page with a one-line verdict per model. + `scripts/make_results_pages.py` generates every number and chart these + pages cite (`docs/assets/results/`), by running + `scripts/benchmark_models_vs_baselines.py` and one worked example of + `score_upgrade_prospects`, so nothing on the pages is typed by hand. - `scripts/benchmark_models_vs_baselines.py`: pairs every estimator with the simple domain rule it is meant to replace (rank by last year's total, predict last gift, mail everyone) and evaluates both on held-out, walk-forward splits: top-1%/5%/10% hit rate and lift, ROC-AUC, average precision and decile calibration for classifiers; MAE and within-25% for amount predictions; net revenue/ROI for cost-aware mail selection. Runs on the synthetic donor panel (five seeds, mean and min-max) by default; `--skip-kdd98` stays fully offline, `--fast` gives a one-seed smoke run, and `--out` writes the results table to JSON and CSV. Separate from the existing `scripts/benchmark_models.py` per-model accuracy table, which this does not replace or touch. - `philanthropy.ingest.map_columns(df, mapping, *, required=...)`: renames a user-supplied export's headers to canonical names and raises one diff --git a/docs/assets/results/ask_kdd98.png b/docs/assets/results/ask_kdd98.png new file mode 100644 index 0000000..e7656f3 Binary files /dev/null and b/docs/assets/results/ask_kdd98.png differ diff --git a/docs/assets/results/lapse_kdd98.png b/docs/assets/results/lapse_kdd98.png new file mode 100644 index 0000000..2711c9b Binary files /dev/null and b/docs/assets/results/lapse_kdd98.png differ diff --git a/docs/assets/results/planned_giving.png b/docs/assets/results/planned_giving.png new file mode 100644 index 0000000..23b989d Binary files /dev/null and b/docs/assets/results/planned_giving.png differ diff --git a/docs/assets/results/response.png b/docs/assets/results/response.png new file mode 100644 index 0000000..fef97da Binary files /dev/null and b/docs/assets/results/response.png differ diff --git a/docs/assets/results/results.json b/docs/assets/results/results.json new file mode 100644 index 0000000..1ab6954 --- /dev/null +++ b/docs/assets/results/results.json @@ -0,0 +1,186 @@ +{ + "response_synthetic": { + "top1pct": { + "model": 83.33333333333334, + "rule": 90.66666666666669, + "random": 44.18666666666667 + }, + "top5pct": { + "model": 79.86666666666666, + "rule": 83.33333333333333, + "random": 44.18666666666667 + }, + "top10pct": { + "model": 77.26666666666667, + "rule": 77.53333333333335, + "random": 44.18666666666667 + }, + "verdict": "loses" + }, + "lapse_synthetic": { + "top1pct": { + "model": 77.33333333333331, + "rule": 62.66666666666667, + "random": 55.81333333333334 + }, + "top5pct": { + "model": 73.6, + "rule": 67.33333333333333, + "random": 55.81333333333334 + }, + "top10pct": { + "model": 72.46666666666665, + "rule": 68.13333333333334, + "random": 55.81333333333334 + }, + "verdict": "modest" + }, + "ask_synthetic": { + "within25pct_model": 19.616502747914726, + "within25pct_last_gift": 10.406748314067423, + "verdict": "wins" + }, + "upgrade_synthetic": { + "top1pct": { + "model": 36.0, + "rule": 20.727272727272727, + "random": 12.469678378527826 + }, + "top5pct": { + "model": 25.616331357571248, + "rule": 16.46156479040576, + "random": 12.469678378527826 + }, + "top10pct": { + "model": 23.72975556937821, + "rule": 17.54004181901756, + "random": 12.469678378527826 + }, + "verdict": "wins" + }, + "planned_giving_synthetic": { + "n_test": 3000, + "model": { + "0.01": 0.8, + "0.05": 0.7946666666666667, + "0.1": 0.7486666666666666 + }, + "random": { + "0.01": 0.4418666666666667, + "0.05": 0.4418666666666667, + "0.1": 0.4418666666666667 + } + }, + "upgrade_worked_example": { + "validation_fiscal_year": 2022, + "n_validation_rows": 1089, + "top_n": 109, + "model_upgrade_rate_top_n": 0.1926605504587156, + "baseline_topn_fy_total_upgrade_rate": 0.12844036697247707, + "overall_upgrade_rate": 0.12029384756657484, + "deciles": [ + { + "decile": 1, + "n": 108, + "actual_rate": 0.19444444444444445, + "mean_predicted": 0.18023017194654117 + }, + { + "decile": 2, + "n": 109, + "actual_rate": 0.11009174311926606, + "mean_predicted": 0.15718379916529487 + }, + { + "decile": 3, + "n": 109, + "actual_rate": 0.14678899082568808, + "mean_predicted": 0.14634772361494633 + }, + { + "decile": 4, + "n": 109, + "actual_rate": 0.11926605504587157, + "mean_predicted": 0.13953595179006095 + }, + { + "decile": 5, + "n": 109, + "actual_rate": 0.12844036697247707, + "mean_predicted": 0.13427069520377125 + }, + { + "decile": 6, + "n": 109, + "actual_rate": 0.045871559633027525, + "mean_predicted": 0.1290397525226347 + }, + { + "decile": 7, + "n": 109, + "actual_rate": 0.11009174311926606, + "mean_predicted": 0.12330523239616623 + }, + { + "decile": 8, + "n": 109, + "actual_rate": 0.12844036697247707, + "mean_predicted": 0.11765028935587177 + }, + { + "decile": 9, + "n": 109, + "actual_rate": 0.10091743119266056, + "mean_predicted": 0.11148766196263095 + }, + { + "decile": 10, + "n": 109, + "actual_rate": 0.11926605504587157, + "mean_predicted": 0.0991965712056734 + } + ], + "n_scored": 981 + }, + "response_kdd98": { + "top1pct": { + "model": 8.041958041958042, + "rule": 6.293706293706294 + }, + "top5pct": { + "model": 10.272536687631026, + "rule": 6.149545772187281 + }, + "top10pct": { + "model": 8.560447239692524, + "rule": 5.83508036338225 + }, + "verdict": "wins" + }, + "lapse_kdd98": { + "base_rate_pct": 94.92411855951033, + "top1pct": { + "model": 96.33123689727464, + "rule": 94.65408805031447 + }, + "top5pct": { + "model": 96.62544539928736, + "rule": 95.05344791448334 + }, + "top10pct": { + "model": 96.15344303532125, + "rule": 95.01100513573 + } + }, + "ask_kdd98": { + "within25pct_model": 59.600825877494835, + "within25pct_last_gift": 66.13902271163111, + "verdict": "loses" + }, + "who_to_mail_kdd98": { + "net_revenue_model": 4381.860000000001, + "net_revenue_mail_everyone": 3149.0499999999993, + "verdict": "wins", + "note": "mail if E[gift]>$0.68; pieces=18588/28624" + } +} \ No newline at end of file diff --git a/docs/assets/results/upgrade_deciles.png b/docs/assets/results/upgrade_deciles.png new file mode 100644 index 0000000..6593d66 Binary files /dev/null and b/docs/assets/results/upgrade_deciles.png differ diff --git a/docs/assets/results/upgrade_topn.png b/docs/assets/results/upgrade_topn.png new file mode 100644 index 0000000..bdfa1b5 Binary files /dev/null and b/docs/assets/results/upgrade_topn.png differ diff --git a/docs/assets/results/who_to_mail.png b/docs/assets/results/who_to_mail.png new file mode 100644 index 0000000..a8e2627 Binary files /dev/null and b/docs/assets/results/who_to_mail.png differ diff --git a/docs/results/ask.md b/docs/results/ask.md new file mode 100644 index 0000000..fe333cc --- /dev/null +++ b/docs/results/ask.md @@ -0,0 +1,23 @@ +# Suggested ask + +Featured on a real donor file: [KDD Cup 1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html). +We pretended it was 1 June 1997 and asked the model to suggest an amount for +each donor who went on to give. + +Of every 100 suggested amounts, 60 landed within 25% of what the donor +actually gave. Simply asking for what they gave last time landed within 25% +for 66 out of every 100. + +![How close is the suggested ask: model vs. ask-what-they-gave-last-time](../assets/results/ask_kdd98.png) + +**Does not beat the simple rule; use the rule instead.** On this file, asking +a donor for what they gave last time is a better guess than the model's +suggestion. + +## Which data + +Real file: [KDD Cup 1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html). +On our sample (synthetic) donor panel the model did beat "ask what they gave +last time" (about 20 in 100 within 25%, versus about 10 in 100 for the simple +rule), so the answer is not the same on every file. Results on your own data +will differ from both. diff --git a/docs/results/index.md b/docs/results/index.md new file mode 100644 index 0000000..bf37b10 --- /dev/null +++ b/docs/results/index.md @@ -0,0 +1,30 @@ +# Results + +Plain-language answers to one question: **does each model actually help you pick +better than the simple rule your shop already uses, or than picking at random?** +No statistics jargon here; everything is in donor counts. For the numbers behind +these pages, see [Model Validation & Benchmarks](../explanation/benchmarks.md). + +| Model | Question it answers | Verdict | +|---|---|---| +| [$1K upgrade](upgrade.md) | Which mid-level donors are about to become $1,000+ donors? | Beats the simple rule | +| [Response](response.md) | Who is most likely to give again next year? | Does not beat the simple rule; use the rule instead | +| [Lapse](lapse.md) | Which donors are about to stop giving? | About the same as the simple rule on our sample data; about the same as random on a real donor file | +| [Suggested ask](ask.md) | How much should we ask a donor for? | Beats "ask what they gave last time" on our sample data; loses to it on a real donor file | +| [Planned giving](planned_giving.md) | Which donors look like bequest prospects? | Not yet tested on real bequest data | +| [Who to mail](who_to_mail.md) | Is it worth mailing this donor at all? | Beats mailing everyone | + +## How we tested each one + +For every model we picked a cutoff date, gave the model only the gifts recorded +up to that date, and then checked what the same donors actually did in the +following fiscal year. Every model is compared against a simple rule a +fundraising shop already uses without any model (rank by past giving, ask for +what they gave last time, mail everyone) and against picking at random. We ran +this on two kinds of data: a synthetic sample donor panel we generate +ourselves (five different random draws, averaged, so one lucky sample can't +flatter the numbers), and a real public file, [KDD Cup +1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html), a 1990s +direct-mail history from a real nonprofit. Every number on these pages is +produced by `scripts/make_results_pages.py`, committed alongside its output in +`docs/assets/results/`, so anyone can regenerate them. diff --git a/docs/results/lapse.md b/docs/results/lapse.md new file mode 100644 index 0000000..3e1458e --- /dev/null +++ b/docs/results/lapse.md @@ -0,0 +1,25 @@ +# Lapse + +Featured on a real donor file: [KDD Cup 1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html). +We pretended it was 1 June 1997, the date of that program's own held-out +mailing, and checked who gave nothing to it. + +Almost everyone in this file was already about to lapse: about 95 out of +every 100 donors gave nothing to the next mailing, whether you pick with a +model or not. Of our top 10% of picks, 96 out of every 100 lapsed. Picking +"whoever gave nothing last time" found 95 out of every 100. Picking at random +also finds about 95 out of every 100. + +![Model vs. gave-nothing-last-time vs. random, top 1/5/10% of picks](../assets/results/lapse_kdd98.png) + +**About the same as random picks.** When almost every donor is a lapse risk, +telling them apart barely matters; do not expect a lapse model to sharpen your +list much on a file shaped like this one. + +## Which data + +Real file: [KDD Cup 1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html), +where lapsing is close to universal. On our sample (synthetic) donor panel, +where lapsing is a genuine minority outcome, the model came out about the same +as the simple rule ("gave nothing last period") rather than the same as +random. Results on your own file will differ from both. diff --git a/docs/results/planned_giving.md b/docs/results/planned_giving.md new file mode 100644 index 0000000..4277be3 --- /dev/null +++ b/docs/results/planned_giving.md @@ -0,0 +1,23 @@ +# Planned giving + +We pretended it was 30 June 2022: the model only saw gifts up to that date, +then we checked which donors actually gave again in fiscal year 2023. Our +sample data has no real bequest-intent label to test against (no fundraising +file we ship does), so this checks the next best thing: does the score beat +random guessing on a related outcome? There is also no established simple +rule for planned-giving prospects to compare against, the way "rank by giving +so far" exists for major gifts. + +Of our top 10% of picks, 75 out of every 100 gave again next year. Picking at +random finds about 44 out of every 100. + +![Model vs. random, top 1/5/10% of picks](../assets/results/planned_giving.png) + +**Not yet tested on real bequest data.** The score separates repeat donors +better than chance, but that is not the same as finding bequest prospects. +Treat it as unproven until it is checked against known planned gifts. + +## Which data + +Sample (synthetic) donor panel, five random draws averaged; no real +planned-giving data was used. Results on your own bequest data will differ. diff --git a/docs/results/response.md b/docs/results/response.md new file mode 100644 index 0000000..1e8e7db --- /dev/null +++ b/docs/results/response.md @@ -0,0 +1,23 @@ +# Response (will they give again?) + +We pretended it was 30 June 2022: the model only saw gifts up to that date, +then we checked which donors actually gave again in fiscal year 2023 (1 July +2022 to 30 June 2023). + +Of our top 5% of picks, 80 out of every 100 gave again. Ranking by total +giving so far found 83 out of every 100. Picking at random finds about 44 out +of every 100. + +![Model vs. rank-by-giving vs. random, top 1/5/10% of picks](../assets/results/response.png) + +**Does not beat the simple rule; use the rule instead.** Ranking donors by +their own giving history does at least as well as the model at every pick +size we checked. + +## Which data + +Sample (synthetic) donor panel, five random draws averaged. On the real +public file, [KDD Cup 1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html), +the model did beat its comparison rule (ranking by lifetime giving alone), +so which one wins can depend on your data; results on your own file will +differ from both of these. diff --git a/docs/results/upgrade.md b/docs/results/upgrade.md new file mode 100644 index 0000000..bcf60a8 --- /dev/null +++ b/docs/results/upgrade.md @@ -0,0 +1,38 @@ +# $1K upgrade + +We pretended it was 30 June 2021: the model only saw gifts up to that date, +then we checked what mid-level donors actually did in fiscal year 2022 (1 July +2021 to 30 June 2022), specifically which of them crossed $1,000 for the +first time. + +Of our top 10% of picks, 24 out of every 100 crossed $1,000. Ranking by this +year's giving total alone found 18 out of every 100. Picking at random finds +about 12 out of every 100. + +![Model vs. rank-by-total vs. random, top 1/5/10% of picks](../assets/results/upgrade_topn.png) + +**Beats the simple rule.** + +## Worked example + +Run `score_upgrade_prospects` on a sample donor panel +(`make_donor_panel(random_state=0)`), cutting off at 30 June 2021 and checking +fiscal year 2022. Out of 1,089 mid-level donors held out for validation, the +model's top 109 picks (its top 10%) included 21 who actually upgraded. +Ranking those same 1,089 donors by this year's total giving instead would have +found 14. Picking 109 of them at random would find about 13. + +The chart below breaks the same validation fold into ten equal-sized groups by +model score (D1 = the 10% the model liked most, D10 = the 10% it liked least), +and it shows where the gain comes from: the model's top 10% clearly stands +out, while the groups below it sit close to the overall rate of about 12 in +100. Use the model to pick your top slice, not to rank the whole file: + +![Upgrade rate by decile](../assets/results/upgrade_deciles.png) + +## Which data + +Sample (synthetic) donor panel, five random draws averaged. We do not show a +$1,000-upgrade result on KDD Cup 1998: that file's test year has almost no +qualifying gifts to check against, so any number there would be noise, not a +result. Results on your own file will differ from both. diff --git a/docs/results/who_to_mail.md b/docs/results/who_to_mail.md new file mode 100644 index 0000000..2806a01 --- /dev/null +++ b/docs/results/who_to_mail.md @@ -0,0 +1,22 @@ +# Who to mail + +Featured on a real donor file: [KDD Cup 1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html). +We pretended it was 1 June 1997 and asked: for each donor, is a $0.68 mailing +worth sending, or should we skip it? + +We mailed 18,588 of the 28,624 donors held out for this test, only the ones +where the model expected the gift to beat the mailing cost. That brought in +$4,382 after mailing costs. Mailing all 28,624 of them would have brought in +$3,149 after costs. + +![Net revenue: mail-only-likely-responders vs. mail-everyone](../assets/results/who_to_mail.png) + +**Beats mailing everyone.** Skipping the donors least likely to respond raised +more money net, not less, even though fewer pieces went out. + +## Which data + +Real file: [KDD Cup 1998](https://kdd.ics.uci.edu/databases/kddcup98/kddcup98.html), +a 1990s direct-mail history; gift sizes there are small (a few dollars to a +few hundred), so its dollar figures will not resemble a major-gift program. +Results on your own file, and at your own mailing cost, will differ. diff --git a/mkdocs.yml b/mkdocs.yml index dff65ec..f3de3a7 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -124,6 +124,14 @@ nav: - how-to/estimate_appeal_uplift.md - how-to/save_and_load_models.md - how-to/develop_and_test.md + - Results: + - results/index.md + - results/upgrade.md + - results/response.md + - results/lapse.md + - results/ask.md + - results/planned_giving.md + - results/who_to_mail.md - Explanation: - explanation/index.md - explanation/fundraising_metrics.md diff --git a/scripts/make_results_pages.py b/scripts/make_results_pages.py new file mode 100644 index 0000000..224c3ef --- /dev/null +++ b/scripts/make_results_pages.py @@ -0,0 +1,390 @@ +""" +scripts/make_results_pages.py +============================== +Generates every number and chart the docs "Results" section cites. + +It never types a result by hand: it imports and calls the ``bench_*`` +functions in ``scripts/benchmark_models_vs_baselines.py`` (the committed, +reproducible model-vs-baseline benchmark), adds a "random pick" baseline for +each hit-rate comparison (the expected hit rate of picking that many donors +uniformly at random, which is just the fold's own positive rate, computed +from the same train/test construction), and runs one worked example of +``score_upgrade_prospects`` on ``make_donor_panel(random_state=0)`` for the +$1K-upgrade page's donor counts and decile chart. + +Writes ``docs/assets/results/results.json`` (every number, for the docs pages +to cite) and one PNG chart per results page into ``docs/assets/results/``. + +Run: + + python scripts/make_results_pages.py # synthetic only + python scripts/make_results_pages.py --with-kdd98 # + KDD Cup 1998 (downloads ~36MB) +""" + +from __future__ import annotations + +import argparse +import json +import sys +import warnings +from pathlib import Path +from typing import Any, Dict, List + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 +import numpy as np # noqa: E402 + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) +import benchmark_models_vs_baselines as bm # noqa: E402 + +from philanthropy.datasets import make_donor_panel # noqa: E402 +from philanthropy.models import PlannedGivingIntentScorer, score_upgrade_prospects # noqa: E402 + +OUT_DIR = ROOT / "docs" / "assets" / "results" +OUT_DIR.mkdir(parents=True, exist_ok=True) + +SEEDS = bm.DEFAULT_SEEDS +N_DONORS, N_YEARS = 3000, 7 + +# Categorical palette slots 1/2/3 (blue/orange/aqua) from the dataviz skill's +# validated default palette: model, simple rule, random. +COLOR_MODEL = "#2a78d6" +COLOR_RULE = "#eb6834" +COLOR_RANDOM = "#898781" # muted ink: "random" is a floor, not a series +INK_PRIMARY = "#0b0b0b" +INK_SECONDARY = "#52514e" +GRID = "#e1e0d9" + + +def _bar_chart( + path: Path, + groups: List[str], + series: Dict[str, List[float]], + colors: Dict[str, str], + ylabel: str, + title: str, +) -> None: + """Grouped bar chart: one group of bars per `groups` entry, one bar per + series. Thin bars, direct value labels, recessive gridlines, a legend.""" + n_groups, n_series = len(groups), len(series) + width = 0.8 / n_series + x = np.arange(n_groups) + fig, ax = plt.subplots(figsize=(6.4, 4.2), dpi=150) + ax.set_facecolor("#fcfcfb") + fig.patch.set_facecolor("#fcfcfb") + for i, (name, values) in enumerate(series.items()): + offset = (i - (n_series - 1) / 2) * width + bars = ax.bar( + x + offset, values, width=width * 0.9, label=name, + color=colors[name], edgecolor="none", + ) + for b, v in zip(bars, values): + ax.text( + b.get_x() + b.get_width() / 2, b.get_height(), f"{v:.0f}", + ha="center", va="bottom", fontsize=9, color=INK_PRIMARY, + ) + ax.set_xticks(x) + ax.set_xticklabels(groups) + ax.set_ylabel(ylabel, color=INK_SECONDARY) + ax.set_title(title, color=INK_PRIMARY, fontsize=11, loc="left") + ax.spines[["top", "right"]].set_visible(False) + ax.spines[["left", "bottom"]].set_color(GRID) + ax.tick_params(colors=INK_SECONDARY) + ax.yaxis.grid(True, color=GRID, linewidth=0.8) + ax.set_axisbelow(True) + ax.legend(frameon=False, labelcolor=INK_SECONDARY) + fig.tight_layout() + fig.savefig(path) + plt.close(fig) + + +def _row(rows, model, metric): + for r in rows: + if r.model == model and r.metric == metric: + return r + return None + + +# --------------------------------------------------------------------------- # +# Random-pick baselines (expected hit rate of a uniform-random pick = the +# fold's own positive rate). Reuses the exact train/test construction the +# benchmark script uses, so this is not a separately-typed number. +# --------------------------------------------------------------------------- # +def random_rate_response() -> float: + rates = [] + for seed in SEEDS: + _, test = bm._train_test_periods(bm._period_panel(N_DONORS, N_YEARS, seed)) + rates.append(test["y_response"].mean()) + return float(np.mean(rates)) + + +def random_rate_lapse() -> float: + rates = [] + for seed in SEEDS: + _, test = bm._train_test_periods(bm._period_panel(N_DONORS, N_YEARS, seed)) + rates.append(1.0 - test["y_response"].mean()) + return float(np.mean(rates)) + + +def random_rate_upgrade(threshold: float = 1000.0, band=(100.0, 999.0)) -> float: + rates = [] + for seed in SEEDS: + panel = bm.make_donor_panel(n_donors=N_DONORS, n_years=N_YEARS, random_state=seed) + years = sorted(panel["gifts"]["fiscal_year"].unique()) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + snaps = bm.build_upgrade_snapshots( + panel["gifts"], fiscal_years=years[:-1], threshold=threshold, band=band, + ) + if snaps.empty or snaps["fiscal_year"].nunique() < 2: + continue + fy = snaps["fiscal_year"].to_numpy() + y = snaps["target"].to_numpy() + splitter = bm.FiscalYearGroupedSplitter(n_splits=1, drop_repeat_donors=False) + train_idx, test_idx = list(splitter.split(snaps[["fy_total"]].to_numpy(), groups=fy))[-1] + rates.append(y[test_idx].mean()) + return float(np.mean(rates)) + + +def planned_giving_hit_rates() -> Dict[str, Any]: + """PlannedGivingIntentScorer has no bequest-intent label in the sample + data (see benchmark_models_vs_baselines.bench_planned_giving's own + docstring): this reuses the giving-response label as the nearest + available proxy, purely as a coverage check against random picking, not + a claim about real planned-giving intent.""" + fracs = (0.01, 0.05, 0.10) + model_rates: Dict[float, list] = {f: [] for f in fracs} + random_rates: Dict[float, list] = {f: [] for f in fracs} + n_test = [] + for seed in SEEDS: + train, test = bm._train_test_periods(bm._period_panel(N_DONORS, N_YEARS, seed)) + Xtr = train[["total", "n", "recent"]].to_numpy() + Xte = test[["total", "n", "recent"]].to_numpy() + ytr, yte = train["y_response"].to_numpy(), test["y_response"].to_numpy() + model = PlannedGivingIntentScorer(random_state=seed).fit(Xtr, ytr) + score = model.predict_intent_score(Xte) + n_test.append(len(yte)) + for f in fracs: + rate, _ = bm._topn_rate(yte, score, f) + model_rates[f].append(rate) + random_rates[f].append(yte.mean()) + return { + "n_test": int(np.mean(n_test)), + "model": {str(f): float(np.mean(v)) for f, v in model_rates.items()}, + "random": {str(f): float(np.mean(v)) for f, v in random_rates.items()}, + } + + +# --------------------------------------------------------------------------- # +# $1K upgrade worked example: score_upgrade_prospects on a fixed seeded panel +# --------------------------------------------------------------------------- # +def upgrade_worked_example() -> Dict[str, Any]: + panel = make_donor_panel(n_donors=N_DONORS, n_years=N_YEARS, random_state=0) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + _, report = score_upgrade_prospects(panel["gifts"], random_state=0) + return { + "validation_fiscal_year": report["validation_fiscal_year"], + "n_validation_rows": report["n_validation_rows"], + "top_n": report["top_n"], + "model_upgrade_rate_top_n": report["model_upgrade_rate_top_n"], + "baseline_topn_fy_total_upgrade_rate": report["baseline_topn_fy_total_upgrade_rate"], + "overall_upgrade_rate": report["overall_upgrade_rate"], + "deciles": report["deciles"], + "n_scored": report["n_scored"], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--with-kdd98", action="store_true", help="Also run the KDD Cup 1998 section (downloads ~36MB).") + args = parser.parse_args() + + results: Dict[str, Any] = {} + + # --- synthetic: response / major gift ----------------------------------- + resp_rows = bm.bench_response(SEEDS, N_DONORS, N_YEARS) + rand_resp = random_rate_response() * 100 + results["response_synthetic"] = { + f"top{p}pct": { + "model": _row(resp_rows, "MajorGiftClassifier", f"top{p}pct_hit_rate").value * 100, + "rule": _row(resp_rows, "MajorGiftClassifier", f"top{p}pct_hit_rate").baseline * 100, + "random": rand_resp, + } + for p in (1, 5, 10) + } + results["response_synthetic"]["verdict"] = _row(resp_rows, "MajorGiftClassifier", "top10pct_hit_rate").verdict + _bar_chart( + OUT_DIR / "response.png", + ["Top 1%", "Top 5%", "Top 10%"], + { + "Model": [results["response_synthetic"][f"top{p}pct"]["model"] for p in (1, 5, 10)], + "Rank by giving so far": [results["response_synthetic"][f"top{p}pct"]["rule"] for p in (1, 5, 10)], + "Random pick": [results["response_synthetic"][f"top{p}pct"]["random"] for p in (1, 5, 10)], + }, + {"Model": COLOR_MODEL, "Rank by giving so far": COLOR_RULE, "Random pick": COLOR_RANDOM}, + ylabel="Gave next year, out of every 100 picked (%)", + title="Who responds next year: model vs. rank-by-giving vs. random", + ) + + # --- synthetic: lapse ---------------------------------------------------- + lapse_rows = bm.bench_lapse(SEEDS, N_DONORS, N_YEARS) + rand_lapse = random_rate_lapse() * 100 + results["lapse_synthetic"] = { + f"top{p}pct": { + "model": _row(lapse_rows, "LapsePredictor", f"top{p}pct_hit_rate").value * 100, + "rule": _row(lapse_rows, "LapsePredictor", f"top{p}pct_hit_rate").baseline * 100, + "random": rand_lapse, + } + for p in (1, 5, 10) + } + results["lapse_synthetic"]["verdict"] = _row(lapse_rows, "LapsePredictor", "top10pct_hit_rate").verdict + + # --- synthetic: ask ------------------------------------------------- + ask_rows = bm.bench_ask(SEEDS, N_DONORS, N_YEARS) + within = _row(ask_rows, "AskAmountRecommender", "within25pct") + results["ask_synthetic"] = { + "within25pct_model": within.value * 100, + "within25pct_last_gift": within.baseline * 100, + "verdict": within.verdict, + } + + # --- synthetic: $1K upgrade (bench_upgrade) ------------------------------ + upgrade_rows = bm.bench_upgrade(SEEDS, N_DONORS, N_YEARS) + rand_upgrade = random_rate_upgrade() * 100 + results["upgrade_synthetic"] = { + f"top{p}pct": { + "model": _row(upgrade_rows, "upgrade_model (MajorGiftClassifier)", f"top{p}pct_hit_rate").value * 100, + "rule": _row(upgrade_rows, "upgrade_model (MajorGiftClassifier)", f"top{p}pct_hit_rate").baseline * 100, + "random": rand_upgrade, + } + for p in (1, 5, 10) + } + results["upgrade_synthetic"]["verdict"] = _row( + upgrade_rows, "upgrade_model (MajorGiftClassifier)", "top10pct_hit_rate" + ).verdict + _bar_chart( + OUT_DIR / "upgrade_topn.png", + ["Top 1%", "Top 5%", "Top 10%"], + { + "Model": [results["upgrade_synthetic"][f"top{p}pct"]["model"] for p in (1, 5, 10)], + "Rank by this year's total": [results["upgrade_synthetic"][f"top{p}pct"]["rule"] for p in (1, 5, 10)], + "Random pick": [results["upgrade_synthetic"][f"top{p}pct"]["random"] for p in (1, 5, 10)], + }, + {"Model": COLOR_MODEL, "Rank by this year's total": COLOR_RULE, "Random pick": COLOR_RANDOM}, + ylabel="Crossed $1,000 next year, out of every 100 picked (%)", + title="Who upgrades to $1,000+: model vs. rank-by-total vs. random", + ) + + # --- synthetic: planned giving (coverage vs. chance) --------------------- + pg = planned_giving_hit_rates() + results["planned_giving_synthetic"] = pg + _bar_chart( + OUT_DIR / "planned_giving.png", + ["Top 1%", "Top 5%", "Top 10%"], + { + "Model": [pg["model"][str(f)] * 100 for f in (0.01, 0.05, 0.10)], + "Random pick": [pg["random"][str(f)] * 100 for f in (0.01, 0.05, 0.10)], + }, + {"Model": COLOR_MODEL, "Random pick": COLOR_RANDOM}, + ylabel="Gave next year, out of every 100 picked (%)", + title="Planned-giving score vs. random (giving-response used as a stand-in label)", + ) + + # --- $1K upgrade worked example ------------------------------------------ + ex = upgrade_worked_example() + results["upgrade_worked_example"] = ex + deciles = ex["deciles"] + _bar_chart( + OUT_DIR / "upgrade_deciles.png", + [f"D{d['decile']}" for d in deciles], + {"Upgrade rate": [d["actual_rate"] * 100 if d["actual_rate"] is not None else 0.0 for d in deciles]}, + {"Upgrade rate": COLOR_MODEL}, + ylabel="Crossed $1,000 next year (%)", + title="Upgrade rate by decile (D1 = top 10% of picks, D10 = bottom 10%)", + ) + + # --- KDD98 (opt-in) -------------------------------------------------- + if args.with_kdd98: + seed = bm.KDD_SEED + kdd_resp = bm.bench_kdd_response(seed) + kdd_lapse = bm.bench_kdd_lapse(seed) + kdd_ask = bm.bench_kdd_ask(seed) + kdd_cost = bm.bench_kdd_cost_aware(seed) + + results["response_kdd98"] = { + f"top{p}pct": { + "model": _row(kdd_resp, "MajorGiftClassifier", f"top{p}pct_hit_rate").value * 100, + "rule": _row(kdd_resp, "MajorGiftClassifier", f"top{p}pct_hit_rate").baseline * 100, + } + for p in (1, 5, 10) + } + results["response_kdd98"]["verdict"] = _row(kdd_resp, "MajorGiftClassifier", "top10pct_hit_rate").verdict + + lapse_top = {p: _row(kdd_lapse, "LapsePredictor", f"top{p}pct_hit_rate") for p in (1, 5, 10)} + donors = bm.fetch_kdd98_donors() + base_rate_lapse_kdd = float((donors["TARGET_B"].to_numpy() == 0).mean()) * 100 + results["lapse_kdd98"] = { + "base_rate_pct": base_rate_lapse_kdd, + **{f"top{p}pct": {"model": r.value * 100, "rule": r.baseline * 100} for p, r in lapse_top.items()}, + } + _bar_chart( + OUT_DIR / "lapse_kdd98.png", + ["Top 1%", "Top 5%", "Top 10%"], + { + "Model": [results["lapse_kdd98"][f"top{p}pct"]["model"] for p in (1, 5, 10)], + "Gave nothing last time": [results["lapse_kdd98"][f"top{p}pct"]["rule"] for p in (1, 5, 10)], + "Random pick": [base_rate_lapse_kdd] * 3, + }, + {"Model": COLOR_MODEL, "Gave nothing last time": COLOR_RULE, "Random pick": COLOR_RANDOM}, + ylabel="Lapsed next period, out of every 100 picked (%)", + title="Who lapses next: KDD Cup 1998 (almost everyone lapses here)", + ) + + ask_row = _row(kdd_ask, "AskAmountRecommender", "within25pct") + results["ask_kdd98"] = { + "within25pct_model": ask_row.value * 100, + "within25pct_last_gift": ask_row.baseline * 100, + "verdict": ask_row.verdict, + } + _bar_chart( + OUT_DIR / "ask_kdd98.png", + ["Suggested ask"], + { + "Model": [results["ask_kdd98"]["within25pct_model"]], + "Ask what they gave last time": [results["ask_kdd98"]["within25pct_last_gift"]], + }, + {"Model": COLOR_MODEL, "Ask what they gave last time": COLOR_RULE}, + ylabel="Within 25% of what the donor actually gave (%)", + title="How close is the suggested ask: KDD Cup 1998", + ) + + net_row = _row(kdd_cost, "cost_aware_selection", "net_revenue") + results["who_to_mail_kdd98"] = { + "net_revenue_model": net_row.value, + "net_revenue_mail_everyone": net_row.baseline, + "verdict": net_row.verdict, + "note": net_row.note, + } + _bar_chart( + OUT_DIR / "who_to_mail.png", + ["Net revenue"], + { + "Only mail likely responders": [results["who_to_mail_kdd98"]["net_revenue_model"]], + "Mail everyone": [results["who_to_mail_kdd98"]["net_revenue_mail_everyone"]], + }, + {"Only mail likely responders": COLOR_MODEL, "Mail everyone": COLOR_RULE}, + ylabel="Net revenue after mailing cost ($)", + title="Who to mail: KDD Cup 1998", + ) + + with open(OUT_DIR / "results.json", "w") as fh: + json.dump(results, fh, indent=2) + print(f"Wrote {OUT_DIR / 'results.json'} and PNGs to {OUT_DIR}") + + +if __name__ == "__main__": + main()