"""Chapter 28.5 and exercise 28.2: what a run of zero-failure tests can support. If n independent tests drawn from one fixed distribution all pass, the one-sided 95% upper confidence bound on the failure rate p is 1 - 0.05**(1/n) (section 3.7). The rule of three approximates it by 3/n. Inverting the bound gives the number of clean tests needed to support a target rate: n = ln(0.05) / ln(1 - target), rounded up. """ import math ALPHA = 0.05 def upper_bound(n: int) -> float: """One-sided 95% upper bound on the failure rate after n tests with zero failures.""" return 1 - ALPHA ** (1 / n) def tests_needed(target: float) -> int: """Smallest n whose zero-failure upper bound is at or below the target rate.""" n = math.ceil(math.log(ALPHA) / math.log(1 - target)) while upper_bound(n) > target: # guard against rounding at the boundary n += 1 return n print("Zero failures in n tests: one-sided 95% upper bound on the failure rate") for n in (10, 30, 100, 300, 1000, 3000, 10000, 30000): print(f" n = {n:>6}: exact {upper_bound(n):.4%} rule of three 3/n = {3 / n:.4%}") print() print("Clean tests needed to bring the upper bound down to a target rate") for target in (0.03, 0.01, 0.001, 0.0001, 0.00001): n = tests_needed(target) print(f" target {target:.3%} (1 in {round(1 / target):,}): n = {n:,} (rule of three: {round(3 / target):,})") print() # Exercise 28.2 (b): 100 clean tests; how many times would a product with true rate 1/10000 fail # at least once in 100 tests? The chance of seeing zero failures is (1 - p)**n. for p in (0.0001, 0.01, 0.03): print(f" true rate {p:.2%}: P(zero failures in 100 tests) = {(1 - p) ** 100:.1%}") print() # Exercise 28.2 (c): only part of the tests come from the deployment distribution. n_relevant = 20 print(f" only {n_relevant} of the 100 tests cover the deployment situation: bound {upper_bound(n_relevant):.1%}")