"""Chapter 6 (section 6.2 and exercise 6.1): expected return, and what one labeler's choices reveal. Data constructed for teaching. Each of 18 pairs holds two answers to the same bookkeeping question. Every answer is long or short (L/S) and correct or wrong (C/W). Pairs 1-6 differ only in length, pairs 7-12 only in correctness, pairs 13-18 put a long wrong answer against a short correct one. "chosen" is the side ("A" or "B") the labeler preferred. """ from math import comb, exp, log NAMES = {"LC": "长且正确", "LW": "长且错误", "SC": "短且正确", "SW": "短且错误"} pairs = [ # (id, answer A, answer B, chosen) (1, "LC", "SC", "A"), (2, "SC", "LC", "B"), (3, "LW", "SW", "A"), (4, "LC", "SC", "B"), (5, "SW", "LW", "B"), (6, "SW", "LW", "B"), (7, "LC", "LW", "A"), (8, "SW", "SC", "B"), (9, "LW", "LC", "B"), (10, "SC", "SW", "A"), (11, "LC", "LW", "B"), (12, "SW", "SC", "B"), (13, "LW", "SC", "A"), (14, "SC", "LW", "B"), (15, "LW", "SC", "A"), (16, "SC", "LW", "A"), (17, "SC", "LW", "B"), (18, "LW", "SC", "A"), ] def kind(a, b): if a[1] == b[1]: return "length only" if a[0] == b[0]: return "correctness only" return "conflict" def is_long(c): return c[0] == "L" def is_correct(c): return c[1] == "C" # ---- section 6.2: expected return of a one-step policy ------------------------------------- print("section 6.2: expected return") for p_long in (0.7, 1.0, 0.0): er = p_long * 0.8 + (1 - p_long) * 0.5 print(f" P(long) = {p_long:.1f}: expected return = {p_long} x 0.8 + {1 - p_long:.1f} x 0.5 = {er:.2f}") print() # ---- exercise 6.1 ----------------------------------------------------------------------------- print("exercise 6.1: the pairs") print("| 编号 | 回答 A | 回答 B | 标注员选择 |") print("| --- | --- | --- | --- |") for i, a, b, ch in pairs: print(f"| {i} | {NAMES[a]} | {NAMES[b]} | {ch} |") print() stats = {} for i, a, b, ch in pairs: win, lose = (a, b) if ch == "A" else (b, a) k = kind(a, b) s = stats.setdefault(k, {"n": 0, "longer": 0, "correct": 0}) s["n"] += 1 s["longer"] += is_long(win) and not is_long(lose) s["correct"] += is_correct(win) and not is_correct(lose) for k in ("length only", "correctness only", "conflict"): s = stats[k] print(f"{k:17s}: n = {s['n']}, chose longer = {s['longer']}, chose correct = {s['correct']}") len_diff = [p for p in pairs if kind(p[1], p[2]) in ("length only", "conflict")] cor_diff = [p for p in pairs if kind(p[1], p[2]) in ("correctness only", "conflict")] chose_longer = sum(1 for _, a, b, ch in len_diff if is_long(a if ch == "A" else b)) chose_correct = sum(1 for _, a, b, ch in cor_diff if is_correct(a if ch == "A" else b)) print(f"pooled, pairs where length differs: chose longer {chose_longer}/{len(len_diff)} = {chose_longer / len(len_diff):.0%}") print(f"pooled, pairs where correctness differs: chose correct {chose_correct}/{len(cor_diff)} = {chose_correct / len(cor_diff):.0%}") chose_a = sum(1 for *_, ch in pairs if ch == "A") print(f"chose A in {chose_a}/{len(pairs)} pairs") p5of6 = sum(comb(6, j) for j in (5, 6)) / 2**6 print(f"P(at least 5 of 6 | labeler picks at random) = {p5of6:.4f}") print() # ---- the simplest reward model: score = a * long + b * correct -------------------------------- # P(labeler picks A) = 1 / (1 + exp(-(score(A) - score(B)))); fit a, b by gradient ascent on the # log-likelihood of the 18 choices. data = [] for _, a, b, ch in pairs: dl = is_long(a) - is_long(b) dc = is_correct(a) - is_correct(b) data.append((dl, dc, 1 if ch == "A" else 0)) wa, wb = 0.0, 0.0 for _ in range(20000): ga = gb = 0.0 for dl, dc, y in data: prob = 1 / (1 + exp(-(wa * dl + wb * dc))) ga += (y - prob) * dl gb += (y - prob) * dc wa += 0.05 * ga wb += 0.05 * gb loglik = sum(log(1 / (1 + exp(-(wa * dl + wb * dc)))) if y else log(1 - 1 / (1 + exp(-(wa * dl + wb * dc)))) for dl, dc, y in data) print(f"fitted reward weights: long = {wa:.3f}, correct = {wb:.3f}, log-likelihood = {loglik:.3f}") for x, y in (("LW", "SC"), ("LC", "SC"), ("LC", "LW")): sx = wa * is_long(x) + wb * is_correct(x) sy = wa * is_long(y) + wb * is_correct(y) print(f" reward({NAMES[x]}) - reward({NAMES[y]}) = {sx - sy:+.3f}; P(prefer {NAMES[x]}) = {1 / (1 + exp(-(sx - sy))):.3f}") # ---- what only the conflict pairs, or only aligned pairs, could tell -------------------------- print() print("conflict pairs alone: every choice of the long answer is also a choice of the wrong answer,") print("so 'prefers long' and 'prefers wrong' predict exactly the same choices there.")