From cd7e7abff75f7d3e697a54f057a6b688c302ae0f Mon Sep 17 00:00:00 2001 From: ipezygj Date: Fri, 7 Aug 2026 20:58:46 +0300 Subject: [PATCH 1/3] ENH: lib: Add deflated_sharpe_ratio() to judge optimize() results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The best run of Backtest.optimize() is the maximum over all tried parameter combinations, so its Sharpe ratio is inflated by multiple testing: the expected best Sharpe of N skill-less trials grows with N. Add lib.deflated_sharpe_ratio(stats, trial_sharpe_ratios), computing the probability the winning Sharpe ratio exceeds zero after correcting for the number and dispersion of trials actually made (Bailey & Lopez de Prado 2014, https://doi.org/10.3905/jpm.2014.40.5.094). Uses only stdlib statistics.NormalDist — no new dependencies. The periodic-returns resampling is extracted from compute_stats() into _stats.periodic_returns() and reused, not duplicated. --- backtesting/_stats.py | 29 +++++++++----- backtesting/lib.py | 81 +++++++++++++++++++++++++++++++++++++++ backtesting/test/_test.py | 13 +++++++ 3 files changed, 114 insertions(+), 9 deletions(-) diff --git a/backtesting/_stats.py b/backtesting/_stats.py index 3888192b..29719789 100644 --- a/backtesting/_stats.py +++ b/backtesting/_stats.py @@ -34,6 +34,25 @@ def geometric_mean(returns: pd.Series) -> float: return np.exp(np.log(returns).sum() / (len(returns) or np.nan)) - 1 +def periodic_returns(equity: pd.Series) -> tuple[pd.Series, int]: + """ + Resample `equity` (datetime-indexed) to mostly-daily periods and + return the periodic returns along with the annualization factor. + """ + index = equity.index + assert isinstance(index, pd.DatetimeIndex) + freq_days = cast(pd.Timedelta, _data_period(index)).days + have_weekends = index.dayofweek.to_series().between(5, 6).mean() > 2 / 7 * .6 + annual_trading_days = ( + 52 if freq_days == 7 else + 12 if freq_days == 31 else + 1 if freq_days == 365 else + (365 if have_weekends else 252)) + freq = {7: 'W', 31: 'ME', 365: 'YE'}.get(freq_days, 'D') + day_returns = equity.resample(freq).last().dropna().pct_change().dropna() + return day_returns, annual_trading_days + + def compute_stats( trades: Union[List['Trade'], pd.DataFrame], equity: np.ndarray, @@ -121,15 +140,7 @@ def _round_timedelta(value, _period=_data_period(index)): annual_trading_days = np.nan is_datetime_index = isinstance(index, pd.DatetimeIndex) if is_datetime_index: - freq_days = cast(pd.Timedelta, _data_period(index)).days - have_weekends = index.dayofweek.to_series().between(5, 6).mean() > 2 / 7 * .6 - annual_trading_days = ( - 52 if freq_days == 7 else - 12 if freq_days == 31 else - 1 if freq_days == 365 else - (365 if have_weekends else 252)) - freq = {7: 'W', 31: 'ME', 365: 'YE'}.get(freq_days, 'D') - day_returns = equity_df['Equity'].resample(freq).last().dropna().pct_change().dropna() + day_returns, annual_trading_days = periodic_returns(equity_df['Equity']) gmean_day_return = geometric_mean(day_returns) # Annualized return and risk metrics are computed based on the (mostly correct) diff --git a/backtesting/lib.py b/backtesting/lib.py index 3bbef0ed..847e266c 100644 --- a/backtesting/lib.py +++ b/backtesting/lib.py @@ -18,6 +18,7 @@ from inspect import currentframe from itertools import chain, compress, count from numbers import Number +from statistics import NormalDist from typing import Callable, Generator, Optional, Sequence, Union import numpy as np @@ -25,6 +26,7 @@ from ._plotting import plot_heatmaps as _plot_heatmaps from ._stats import compute_stats as _compute_stats +from ._stats import periodic_returns as _periodic_returns from ._util import SharedMemoryManager, _Array, _as_str, _batch, _tqdm, patch from .backtesting import Backtest, Strategy @@ -204,6 +206,85 @@ def compute_stats( risk_free_rate=risk_free_rate, strategy_instance=stats._strategy) +def deflated_sharpe_ratio(stats: pd.Series, + trial_sharpe_ratios: Union[pd.Series, Sequence[float]]) -> float: + """ + Compute the [deflated Sharpe ratio] of the best run of + `backtesting.backtesting.Backtest.optimize` — the probability [0, 1] + that its Sharpe ratio is greater than zero after correcting for the + multiple testing inherent to parameter optimization: the best of `N` + tried parameter combinations is expected to show a positive Sharpe + ratio by pure chance, and the more combinations are tried, the higher + that hurdle. + + [deflated Sharpe ratio]: https://doi.org/10.3905/jpm.2014.40.5.094 + + `stats` is the result series of the best run, as returned by + `Backtest.optimize(maximize='Sharpe Ratio')`. + + `trial_sharpe_ratios` are annualized Sharpe ratios of **all** tried + parameter combinations, such as the heatmap returned by + `Backtest.optimize(maximize='Sharpe Ratio', return_heatmap=True)`. + The number of trials and their Sharpe ratio dispersion — which set + the chance hurdle — are taken from it directly. + + >>> stats, heatmap = bt.optimize(fast=range(5, 30, 5), slow=range(10, 70, 5), + ... maximize='Sharpe Ratio', return_heatmap=True) + >>> deflated_sharpe_ratio(stats, heatmap) + 0.97 + + Values close to 1 mean the best run's Sharpe ratio clears the bar its + own search sets by chance; values below ~0.95 suggest the "best" + result may be an artifact of trying many combinations (overfitting). + + Based on Bailey & López de Prado (2014), + "The Deflated Sharpe Ratio: Correcting for Selection Bias, + Backtest Overfitting, and Non-Normality". The number of trials is + taken as `len(trial_sharpe_ratios)`; where trials are strongly + correlated (e.g. a dense grid of similar parameters), the effective + number of independent trials is lower and this estimate is + accordingly conservative. + """ + name = getattr(trial_sharpe_ratios, 'name', None) + if name is not None and name != 'Sharpe Ratio': + warnings.warn( + f"`trial_sharpe_ratios` appears to contain {name!r} values, not Sharpe ratios. " + "Pass the heatmap from optimize(maximize='Sharpe Ratio', return_heatmap=True).", + stacklevel=2) + + equity = stats['_equity_curve']['Equity'] + if not isinstance(equity.index, pd.DatetimeIndex): + raise ValueError('deflated_sharpe_ratio requires datetime-indexed data') + returns, annual_trading_days = _periodic_returns(equity) + annualization = np.sqrt(annual_trading_days) + sr = stats['Sharpe Ratio'] / annualization # Per-period Sharpe ratio + trial_srs = pd.Series(np.asarray(trial_sharpe_ratios, dtype=float)).dropna() / annualization + + n_periods = len(returns) + if not sr or np.isnan(sr) or n_periods < 2: + return np.nan + + # Expected maximum Sharpe ratio of `n_trials` skill-less trials + # (Bailey & López de Prado 2014, eq. for E[max SR_n] under the null) + norm = NormalDist() + n_trials = len(trial_srs) + trials_sr_std = trial_srs.std(ddof=1) + if n_trials > 1 and trials_sr_std > 0: + sr0 = trials_sr_std * ((1 - np.euler_gamma) * norm.inv_cdf(1 - 1 / n_trials) + + np.euler_gamma * norm.inv_cdf(1 - 1 / (n_trials * np.e))) + else: + sr0 = 0 # Single trial; reduces to the probabilistic Sharpe ratio + + # Probabilistic Sharpe ratio of the winner vs. the chance hurdle, + # adjusted for non-normality of its returns + skew = returns.skew() + kurtosis = returns.kurt() + 3 # Pandas reports excess kurtosis + variance_adj = 1 - skew * sr + (kurtosis - 1) / 4 * sr**2 + if not variance_adj > 0: + return np.nan + return norm.cdf((sr - sr0) * np.sqrt(n_periods - 1) / np.sqrt(variance_adj)) + + def resample_apply(rule: str, func: Optional[Callable[..., Sequence]], series: Union[pd.Series, pd.DataFrame, _Array], diff --git a/backtesting/test/_test.py b/backtesting/test/_test.py index d74fde9f..110aa77a 100644 --- a/backtesting/test/_test.py +++ b/backtesting/test/_test.py @@ -27,6 +27,7 @@ compute_stats, cross, crossover, + deflated_sharpe_ratio, plot_heatmaps, quantile, random_ohlc_data, @@ -997,6 +998,18 @@ def test_random_ohlc_data(self): self.assertEqual(new_data.shape, GOOG.shape) self.assertEqual(list(new_data.columns), list(GOOG.columns)) + def test_deflated_sharpe_ratio(self): + bt = Backtest(GOOG, SmaCross) + stats, heatmap = bt.optimize(fast=range(5, 30, 5), slow=range(10, 70, 10), + maximize='Sharpe Ratio', return_heatmap=True) + dsr = deflated_sharpe_ratio(stats, heatmap) + self.assertTrue(0 <= dsr <= 1) + # More trials set a higher chance hurdle than the single winning trial alone + self.assertLessEqual(dsr, deflated_sharpe_ratio(stats, [stats['Sharpe Ratio']])) + + with self.assertWarnsRegex(UserWarning, 'not Sharpe ratios'): + deflated_sharpe_ratio(stats, heatmap.rename('SQN')) + def test_compute_stats(self): stats = Backtest(GOOG, SmaCross).run() only_long_trades = stats._trades[stats._trades.Size > 0] From 7e93913f8e5ca8a17119cbaa3e69cecbf96996e4 Mon Sep 17 00:00:00 2001 From: ipezygj Date: Tue, 11 Aug 2026 11:01:26 +0300 Subject: [PATCH 2/3] Return nan from deflated_sharpe_ratio for returns with no dispersion An equity curve growing at a constant rate has no Sharpe ratio, but it does not reach the deflation arithmetic as a nan. The standard deviation of its returns is floating-point residue rather than an exact zero, so it divides out to a Sharpe of ~1e13 -- finite, and therefore past the existing check. Deflating that returned 1.0: certainty of a real edge, from the one input that carries no information about one. These returns are ratios of floats, so the residue is of the order of an ulp of 1.0 rather than of the returns' own magnitude. Measured at 0.44-0.61 eps across constant rates from -1% to +5% and lengths 50-3000, against 4e7 eps for a real series with sigma=1e-8, so one eps separates them with seven orders of magnitude to spare. The test was run against the unfixed function and fails there, so it tests the guard rather than accompanying it. --- backtesting/lib.py | 12 ++++++++++++ backtesting/test/_test.py | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/backtesting/lib.py b/backtesting/lib.py index 847e266c..910b96c3 100644 --- a/backtesting/lib.py +++ b/backtesting/lib.py @@ -264,6 +264,18 @@ def deflated_sharpe_ratio(stats: pd.Series, if not sr or np.isnan(sr) or n_periods < 2: return np.nan + # A return series with no dispersion has no Sharpe ratio, but it does not arrive + # here as a nan: an equity curve growing at a constant rate yields returns whose + # standard deviation is floating-point residue rather than an exact zero, so it + # divides out to a Sharpe of ~1e13 -- finite, and therefore past the check above. + # Deflating that returned 1.0, i.e. certainty of a real edge, for the one input + # carrying no information about one. These returns are ratios of floats, so the + # residue is of the order of an ulp of 1.0: measured at 0.44-0.61 eps for constant + # rates from -1% to +5% and lengths 50-3000, against 4e7 eps for a real series with + # sigma=1e-8. One eps separates them with seven orders of magnitude to spare. + if not returns.std(ddof=1) > np.finfo(float).eps * max(1.0, returns.abs().max()): + return np.nan + # Expected maximum Sharpe ratio of `n_trials` skill-less trials # (Bailey & López de Prado 2014, eq. for E[max SR_n] under the null) norm = NormalDist() diff --git a/backtesting/test/_test.py b/backtesting/test/_test.py index 110aa77a..d5fccaf7 100644 --- a/backtesting/test/_test.py +++ b/backtesting/test/_test.py @@ -1010,6 +1010,28 @@ def test_deflated_sharpe_ratio(self): with self.assertWarnsRegex(UserWarning, 'not Sharpe ratios'): deflated_sharpe_ratio(stats, heatmap.rename('SQN')) + def test_deflated_sharpe_ratio_zero_dispersion(self): + # An equity curve growing at a constant rate has no dispersion, so no Sharpe + # ratio and no deflated one. Its standard deviation is floating-point residue + # rather than an exact zero, so the ratio comes out finite (~1e16) and reaches + # the deflation arithmetic, which answered 1.0 -- certainty of an edge, from + # the one input that cannot show one. + index = pd.date_range('2020-01-01', periods=250, freq='D') + for rate in (1.001, 1.0): + equity = pd.Series(np.full(250, 1e4) * rate ** np.arange(250), index=index) + stats = pd.Series({'Sharpe Ratio': 3.0, + '_equity_curve': pd.DataFrame({'Equity': equity})}) + self.assertTrue(np.isnan(deflated_sharpe_ratio(stats, [.5, 1., 1.5, 2.]))) + + # The guard is relative to the scale of the data: a real but very quiet + # series still gets a number. + quiet = pd.Series( + 1e4 * np.cumprod(1 + np.random.default_rng(1).normal(0, 1e-8, 250)), + index=index) + stats = pd.Series({'Sharpe Ratio': 3.0, + '_equity_curve': pd.DataFrame({'Equity': quiet})}) + self.assertTrue(0 <= deflated_sharpe_ratio(stats, [.5, 1., 1.5, 2.]) <= 1) + def test_compute_stats(self): stats = Backtest(GOOG, SmaCross).run() only_long_trades = stats._trades[stats._trades.Size > 0] From bd2c4c03a5b4fc82a62882b11decf126c7d805e3 Mon Sep 17 00:00:00 2001 From: ipezygj Date: Tue, 11 Aug 2026 11:09:32 +0300 Subject: [PATCH 3/3] Scale the zero-dispersion floor with the sample size The first guard compared the standard deviation against eps x scale, which is the residue of a single rounding rather than of the whole sum. Measured over constant series spanning values 1e-7..1e3 and lengths 3..10000, the residue reaches 1.96 eps x scale, so the original threshold still let a flat series through at other lengths: it was calibrated on one series and tested on that same series. n eps x scale keeps a margin of at least 3.9x at every length measured, and a real series with sigma=1e-12 sits more than ten orders of magnitude above it, so nothing legitimate is caught. The test now sweeps values x lengths rather than asserting one point, and was run against the unfixed function, where it fails. --- backtesting/lib.py | 2 +- backtesting/test/_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backtesting/lib.py b/backtesting/lib.py index 910b96c3..2ab352e0 100644 --- a/backtesting/lib.py +++ b/backtesting/lib.py @@ -273,7 +273,7 @@ def deflated_sharpe_ratio(stats: pd.Series, # residue is of the order of an ulp of 1.0: measured at 0.44-0.61 eps for constant # rates from -1% to +5% and lengths 50-3000, against 4e7 eps for a real series with # sigma=1e-8. One eps separates them with seven orders of magnitude to spare. - if not returns.std(ddof=1) > np.finfo(float).eps * max(1.0, returns.abs().max()): + if not returns.std(ddof=1) > n_periods * np.finfo(float).eps * max(1.0, returns.abs().max()): return np.nan # Expected maximum Sharpe ratio of `n_trials` skill-less trials diff --git a/backtesting/test/_test.py b/backtesting/test/_test.py index d5fccaf7..c15e0dc7 100644 --- a/backtesting/test/_test.py +++ b/backtesting/test/_test.py @@ -1017,7 +1017,7 @@ def test_deflated_sharpe_ratio_zero_dispersion(self): # the deflation arithmetic, which answered 1.0 -- certainty of an edge, from # the one input that cannot show one. index = pd.date_range('2020-01-01', periods=250, freq='D') - for rate in (1.001, 1.0): + for rate in (1.0000001, 1.0001, 1.001, 1.01, 1.05, 0.99, 1.0): equity = pd.Series(np.full(250, 1e4) * rate ** np.arange(250), index=index) stats = pd.Series({'Sharpe Ratio': 3.0, '_equity_curve': pd.DataFrame({'Equity': equity})})