"""Chapter 4 (section 4.4 and exercise 4.1): why choosing among many settings on the test set inflates the score. Constructed setting: 50 hyperparameter settings that are in truth equally good, each with true accuracy 80% on new emails. Each is scored on the same kind of test set of 200 emails. Treat each setting's number of correct answers as an independent Binomial(200, 0.8) draw (a simplification: on a shared test set the scores are correlated, which shrinks the effect but does not remove it). The script computes, exactly, the distribution of the best of the 50 test scores (section 4.4), and of the best of 31 (exercise 4.1: a default setting plus 30 more). """ from math import comb n, p = 200, 0.8 pmf = [comb(n, k) * p**k * (1 - p) ** (n - k) for k in range(n + 1)] cdf = [] acc = 0.0 for q in pmf: acc += q cdf.append(acc) sd_one = (n * p * (1 - p)) ** 0.5 print(f"one setting: expected accuracy = {p:.1%}, standard deviation = {sd_one / n:.2%}") def best_of(settings, thresholds=()): # P(max <= k) = F(k)^settings max_cdf = [c**settings for c in cdf] max_pmf = [max_cdf[0]] + [max_cdf[k] - max_cdf[k - 1] for k in range(1, n + 1)] expected_max = sum(k * q for k, q in enumerate(max_pmf)) print(f"best of {settings} settings on the test set: expected accuracy = {expected_max / n:.2%}") for thr in thresholds: k = round(thr * n) print(f" P(best test accuracy >= {thr:.0%}) = {1 - max_cdf[k - 1]:.1%}") best_of(50, (0.84, 0.85, 0.86)) # section 4.4 best_of(31) # exercise 4.1: the default setting plus 30 more print(f"on fresh emails the chosen setting is still expected to score {p:.0%}")