"""Chapter 11: metrics of 20 judgments and the expected yield of spot checks (constructed data). Setting (constructed for teaching): an evaluator, a person or a reviewing model, judged 20 changes to the bookkeeping program, saying for each whether it breaks the requirements ("problem") or not ("ok"), with a confidence of high or low. The reference answer comes from a senior programmer who checked every change afterwards. A separate risk filter marked each change high or low risk. Positive class = the evaluator says "problem". false alarm (误报): reference ok, evaluator says problem. false-alarm rate = FP / (reference ok) miss (漏检): reference problem, evaluator says ok. miss rate = FN / (reference problem) An "error" of the evaluator is any judgment that disagrees with the reference. """ from fractions import Fraction from math import comb, sqrt # (id, reference, judgment, confidence, risk) J = [ ("J01", "ok", "ok", "high", "high"), ("J02", "ok", "ok", "high", "low"), ("J03", "problem", "problem", "high", "high"), ("J04", "ok", "ok", "high", "low"), ("J05", "ok", "problem", "low", "high"), ("J06", "ok", "ok", "high", "low"), ("J07", "problem", "problem", "high", "high"), ("J08", "ok", "ok", "high", "low"), ("J09", "problem", "ok", "high", "low"), ("J10", "ok", "ok", "high", "low"), ("J11", "ok", "ok", "high", "low"), ("J12", "problem", "problem", "low", "high"), ("J13", "ok", "ok", "high", "low"), ("J14", "ok", "problem", "low", "low"), ("J15", "ok", "ok", "high", "low"), ("J16", "problem", "problem", "low", "low"), ("J17", "ok", "ok", "high", "low"), ("J18", "problem", "ok", "low", "low"), ("J19", "ok", "ok", "low", "low"), ("J20", "ok", "problem", "low", "low"), ] assert len(J) == 20 tp = [j for j in J if j[1] == "problem" and j[2] == "problem"] fn = [j for j in J if j[1] == "problem" and j[2] == "ok"] fp = [j for j in J if j[1] == "ok" and j[2] == "problem"] tn = [j for j in J if j[1] == "ok" and j[2] == "ok"] P, N = len(tp) + len(fn), len(fp) + len(tn) errors = len(fp) + len(fn) print(f"reference: {P} problem, {N} ok") print(f"TP={len(tp)} FN={len(fn)} {[j[0] for j in fn]} FP={len(fp)} {[j[0] for j in fp]} TN={len(tn)}") print(f"accuracy = {len(tp)+len(tn)}/20 = {(len(tp)+len(tn))/20:.1%}") print(f"false-alarm rate = {len(fp)}/{N} = {len(fp)/N:.1%}") print(f"miss rate = {len(fn)}/{P} = {len(fn)/P:.1%}") flagged = len(tp) + len(fp) print(f"(for comparison, precision of the 'problem' flags = {len(tp)}/{flagged} = {len(tp)/flagged:.1%};" f" 1 - precision = {len(fp)}/{flagged} = {len(fp)/flagged:.1%}, not the false-alarm rate)") # Random spot check of 5 of the 20 judgments, without replacement. k = 5 print() print(f"errors among the 20 judgments: {errors}") print(f"expected errors found in a random check of {k}: {k} x {errors}/20 = {Fraction(k*errors, 20)} = {k*errors/20:.2f}") p_none = Fraction(comb(20 - errors, k), comb(20, k)) print(f"P(no error among the {k}) = C({20-errors},{k})/C(20,{k}) = {comb(20-errors,k)}/{comb(20,k)} = {float(p_none):.3f};" f" P(at least one) = {1-float(p_none):.3f}") print(f"expected misses found: {k} x {len(fn)}/20 = {k*len(fn)/20:.2f}") p_none_fn = Fraction(comb(20 - len(fn), k), comb(20, k)) print(f"P(at least one miss among the {k}) = 1 - {comb(20-len(fn),k)}/{comb(20,k)} = {1-float(p_none_fn):.3f}") # check the expectation by enumerating all C(20,5) subsets' counts (via the hypergeometric pmf) exp = sum(i * comb(errors, i) * comb(20 - errors, k - i) for i in range(0, min(k, errors) + 1)) / comb(20, k) assert abs(exp - k * errors / 20) < 1e-12 # Confidence print() for c in ("high", "low"): sub = [j for j in J if j[3] == c] right = sum(1 for j in sub if j[1] == j[2]) print(f"confidence {c}: {right}/{len(sub)} correct = {right/len(sub):.1%}; errors {[j[0] for j in sub if j[1] != j[2]]}") # Risk-based audit: check the 5 judgments marked high risk print() high = [j for j in J if j[4] == "high"] low = [j for j in J if j[4] == "low"] print(f"high-risk items: {[j[0] for j in high]}") print(f" errors found by auditing them: {[j[0] for j in high if j[1] != j[2]]}; misses found: {[j[0] for j in high if j[1]=='problem' and j[2]=='ok']}") low_err = [j for j in low if j[1] != j[2]] low_fn = [j for j in low if j[1] == "problem" and j[2] == "ok"] print(f"low-risk items: {len(low)}, errors among them {[j[0] for j in low_err]}, misses {[j[0] for j in low_fn]}") print(f"real problems among high-risk {sum(1 for j in high if j[1]=='problem')}, among low-risk {sum(1 for j in low if j[1]=='problem')}") # Mixed audit: 3 highest-risk (here: any 3 of the 5 high) + 2 random from the 15 low-risk m = 2 print(f"mixed audit, 2 random from the {len(low)} low-risk: expected misses found = {m} x {len(low_fn)}/{len(low)} = {m*len(low_fn)/len(low):.3f};" f" P(at least one) = 1 - {comb(len(low)-len(low_fn), m)}/{comb(len(low), m)} = {1 - comb(len(low)-len(low_fn), m)/comb(len(low), m):.3f}") # Section 11.6 worked example: 100 outputs, 10 with problems. print() P2, N2, tp2, fp2 = 10, 90, 8, 9 print(f"11.6 example: accuracy = ({tp2}+{N2-fp2})/100 = {(tp2+N2-fp2)/100:.0%}; false-alarm rate = {fp2}/{N2} = {fp2/N2:.0%}; miss rate = {P2-tp2}/{P2} = {(P2-tp2)/P2:.0%}") print(f"always-ok evaluator: accuracy = {N2}/100 = {N2/100:.0%}; false-alarm rate 0%; miss rate 100%") # Section 11.4: standard error of an accuracy measured on 100 questions (Bowman et al. MMLU unassisted 57.2%) p, n = 0.572, 100 print() print(f"11.4: sqrt(p(1-p)/n) with p={p}, n={n}: {sqrt(p*(1-p)/n):.3f}") # Exercise 11.3, answer 4: Bowman et al. MMLU, per-person accuracy and average confidence, without and with # the model (Table 1 and the confidence figures quoted in 11.6), in percentage points. acc0, acc1, conf0, conf1 = 57.2, 75.4, 54, 80 print(f"ex 11.3: MMLU per-person accuracy {acc0}% -> {acc1}%: {acc1-acc0:.1f} points; " f"average confidence {conf0}% -> {conf1}%: {conf1-conf0} points") # Section 11.8: rare problems need large random samples. 1000 outputs, 10 with problems, check 50 at random. Nn, bad, draw = 1000, 10, 50 print() print(f"11.8: expected problems found = {draw} x {bad}/{Nn} = {draw*bad/Nn}; " f"P(at least one) = 1 - C({Nn-bad},{draw})/C({Nn},{draw}) = {1 - comb(Nn-bad, draw)/comb(Nn, draw):.3f}") # Figure fig-11-random-audit: 3 of the 10 problems look low-risk (illustrative). A random audit of 50 # reaches them with the probability below; an audit of high-risk outputs only never reaches them. hidden = 3 print(f"11.8 figure: {hidden} of the {bad} look low-risk; random audit of {draw}: P(at least one of them) = " f"1 - C({Nn-hidden},{draw})/C({Nn},{draw}) = {1 - comb(Nn-hidden, draw)/comb(Nn, draw):.3f}; high-risk audit only: 0") for k in (100, 200, 300): print(f" random audit of {k}: P(at least one of {bad}) = {1 - comb(Nn-bad, k)/comb(Nn, k):.3f}; " f"P(at least one of the {hidden}) = {1 - comb(Nn-hidden, k)/comb(Nn, k):.3f}")