"""Exercise 12.2: position bias of a model judge (constructed data). A judge compares answer X (from model 1) with answer Y (from model 2) on 12 questions, twice: once with X shown first, once with Y shown first. Each verdict is "first", "second" or "tie". Definitions follow Zheng et al. (2023): a pair is consistent when the same answer (or a tie) wins in both orders; "biased toward first" when the judge picks whichever answer is shown first in both orders, and likewise for "second". """ # (question, verdict when X is first, verdict when Y is first) pairs = [ ("Q1", "first", "second"), # X both times ("Q2", "first", "first"), # whichever is first ("Q3", "second", "first"), # Y both times ("Q4", "first", "second"), # X both times ("Q5", "first", "first"), ("Q6", "first", "first"), ("Q7", "second", "second"), # whichever is second ("Q8", "first", "second"), # X both times ("Q9", "first", "first"), ("Q10", "second", "first"), # Y both times ("Q11", "first", "first"), ("Q12", "first", "second"), # X both times ] def winner(verdict, first, second): return {"first": first, "second": second, "tie": "tie"}[verdict] consistent = toward_first = toward_second = 0 cons_x = cons_y = 0 x_first_wins = {"X": 0, "Y": 0, "tie": 0} y_first_wins = {"X": 0, "Y": 0, "tie": 0} for q, v_xf, v_yf in pairs: w1 = winner(v_xf, "X", "Y") w2 = winner(v_yf, "Y", "X") x_first_wins[w1] += 1 y_first_wins[w2] += 1 if w1 == w2: consistent += 1 cons_x += w1 == "X" cons_y += w1 == "Y" elif v_xf == v_yf == "first": toward_first += 1 elif v_xf == v_yf == "second": toward_second += 1 n = len(pairs) print(f"consistent: {consistent}/{n} = {consistent/n:.1%} (X wins {cons_x}, Y wins {cons_y})") print(f"biased toward first: {toward_first}/{n} = {toward_first/n:.1%}") print(f"biased toward second: {toward_second}/{n} = {toward_second/n:.1%}") print(f"if X were always shown first: {x_first_wins}") print(f"if Y were always shown first: {y_first_wins}") print(f"conservative rule (win only if preferred in both orders, else tie): X {cons_x}, Y {cons_y}, tie {n - cons_x - cons_y}")