"""Chapter 3, section 3.4 (power) and exercise 3.1, part 7: how often a paired comparison sees a real difference. Setting (for teaching): models A and B answer the same n questions; a share r of the questions are discordant (one model right, the other wrong), and A's true accuracy is d higher than B's. The paired difference has standard error about sqrt((r - d^2) / n), the formula used in section 3.4 and in the solution of exercise 3.1. "Seeing the difference" means the two-sided 95% interval of the paired difference excludes 0, which is roughly the same as p < 0.05. The script prints the power (the chance of seeing the difference) by the bell-curve approximation, checks it at n = 200 against an exact calculation with the McNemar exact test, and prints the number of questions needed for 80% power. Standard library only. """ import math Z95 = 1.96 # two-sided 95%, rounded as in the text Z80 = 0.84 # one-sided 80%, rounded as in the text def phi(x): return 0.5 * (1 + math.erf(x / math.sqrt(2))) def power_normal(n, r, d): se = math.sqrt((r - d * d) / n) return (1 - phi(Z95 - d / se)) + phi(-Z95 - d / se) def n_halfwidth(r, d): """Questions needed for the 95% interval to have half-width d (section 3.4, line on sample size).""" return Z95 ** 2 * (r - d * d) / d ** 2 def n_power(r, d, z_power=Z80): return (Z95 + z_power) ** 2 * (r - d * d) / d ** 2 def binom_pmf(k, n, p): return math.comb(n, k) * p ** k * (1 - p) ** (n - k) def mcnemar_exact_p(b, D): """Two-sided exact McNemar p-value: b of D discordant questions favour A, fair coin under the null.""" if D == 0: return 1.0 k = max(b, D - b) tail = sum(math.comb(D, i) for i in range(k, D + 1)) / 2 ** D return min(1.0, 2 * tail) def power_exact(n, r, d): """Exact power of the McNemar exact test at 0.05: the number D of discordant questions is binomial(n, r); given D, the number favouring A is binomial(D, (r + d) / (2r)).""" pa = (r + d) / (2 * r) total = 0.0 for D in range(n + 1): pD = binom_pmf(D, n, r) if pD < 1e-15: continue rej = sum(binom_pmf(b, D, pa) for b in range(D + 1) if mcnemar_exact_p(b, D) <= 0.05) total += pD * rej return total d = 0.02 r = 0.10 print(f"Section 3.4: r = {r:.0%} discordant, true difference d = {d * 100:.0f} points") for n in (200, round(n_halfwidth(r, d)), round(n_power(r, d))): print(f" n = {n:5d}: power (bell curve) = {power_normal(n, r, d):.1%}") print(f" exact McNemar test at n = 200: power = {power_exact(200, r, d):.1%}") print(f" n for half-width 2 points = {n_halfwidth(r, d):.0f}; n for 80% power = {n_power(r, d):.0f}") print(f" ratio (1.96 + 0.84)^2 / 1.96^2 = {(Z95 + Z80) ** 2 / Z95 ** 2:.2f}") print(f" few discordant questions (n = 200, r = 2%): bell curve {power_normal(200, 0.02, d):.1%}, exact McNemar {power_exact(200, 0.02, d):.1%}") print(" n for 80% power at other discordance shares:") for rr in (0.05, 0.10, 0.20, 0.30): print(f" r = {rr:.0%}: half-width n = {n_halfwidth(rr, d):.0f}, 80% power n = {n_power(rr, d):.0f}") print() r = 0.16 # exercise 3.1: 32 of 200 questions discordant print(f"Exercise 3.1, part 7: r = {r:.0%}, d = {d * 100:.0f} points") print(f" power at n = 200 (bell curve) = {power_normal(200, r, d):.1%}") print(f" power at n = 200 (exact McNemar) = {power_exact(200, r, d):.1%}") print(f" n for half-width 2 points = {n_halfwidth(r, d):.0f}; n for 80% power = {n_power(r, d):.0f}")