"""Exercise 12.1: agreement between the GPT-4 judge and human experts on MT-Bench (real data). Data: the Hugging Face dataset lmsys/mt_bench_human_judgments (CC BY 4.0), released with Zheng et al., "Judging LLM-as-a-Judge with MT-Bench and Chatbot Arena" (NeurIPS 2023). It has two splits: `human` (3355 pairwise votes by experts and authors) and `gpt4_pair` (2400 pairwise verdicts by GPT-4). Each record has question_id, model_a, model_b, winner, judge, conversation_a, conversation_b, turn. This script reads only the metadata fields. Extraction rule for the 50 records printed in the book (decided before looking at results): 1. In the `human` split, keep votes whose judge starts with "expert_" (drop "author_") and whose turn is 1. Keep the dataset order (row_idx ascending). 2. Take every 26th record starting from the first (positions 0, 26, 52, ...), 50 records. 3. For each, find the GPT-4 verdict on the same question, the same two models (in either order) and the same turn in the `gpt4_pair` split. Winners are mapped to model names so that the order of model_a/model_b does not matter. "tie (inconsistent)" (GPT-4 changed its verdict when the answers were swapped) counts as a tie. Usage (standard library only; needs network for --download and --show): python3 tools/exercises/ch12_mtbench.py --download DIR # save both splits as JSON in DIR python3 tools/exercises/ch12_mtbench.py --data DIR # run the analysis on the saved copy python3 tools/exercises/ch12_mtbench.py --show ROW_IDX # print the two conversations of one # record of the `human` split The same records can be browsed in the dataset viewer on the Hugging Face page of the dataset. """ import argparse import json import os import time import urllib.request API = ("https://datasets-server.huggingface.co/rows?dataset=lmsys/mt_bench_human_judgments" "&config=default&split={split}&offset={offset}&length=100") SPLITS = {"human": 3355, "gpt4_pair": 2400} FIELDS = ("question_id", "model_a", "model_b", "winner", "judge", "turn") CATEGORY = [(81, "writing"), (91, "roleplay"), (101, "reasoning"), (111, "math"), (121, "coding"), (131, "extraction"), (141, "stem"), (151, "humanities")] def download(directory): os.makedirs(directory, exist_ok=True) for split, n in SPLITS.items(): rows = [] for offset in range(0, n, 100): for attempt in range(12): try: with urllib.request.urlopen(API.format(split=split, offset=offset), timeout=60) as r: page = json.load(r) break except Exception: time.sleep(10) else: raise SystemExit(f"could not fetch {split} at offset {offset}") for item in page["rows"]: rec = {k: item["row"][k] for k in FIELDS} rec["row_idx"] = item["row_idx"] rows.append(rec) with open(os.path.join(directory, f"{split}.json"), "w") as f: json.dump(rows, f) print(f"saved {split}: {len(rows)} rows") def show(row_idx): url = API.format(split="human", offset=row_idx).replace("length=100", "length=1") with urllib.request.urlopen(url, timeout=60) as r: row = json.load(r)["rows"][0]["row"] print(f"row_idx {row_idx}: question {row['question_id']} ({category(row['question_id'])}), turn {row['turn']}, " f"judge {row['judge']}, winner {row['winner']}") for side in ("a", "b"): print(f"\n===== model_{side}: {row['model_' + side]} =====") for msg in row["conversation_" + side]: print(f"[{msg['role']}]\n{msg['content']}\n") def category(qid): name = CATEGORY[0][1] for start, cat in CATEGORY: if qid >= start: name = cat return name def verdict(rec): """Winner as a model name, or 'tie'.""" w = rec["winner"] if w == "model_a": return rec["model_a"] if w == "model_b": return rec["model_b"] return "tie" # 'tie' and 'tie (inconsistent)' def key(rec): return (rec["question_id"], tuple(sorted((rec["model_a"], rec["model_b"]))), rec["turn"]) def agreement(pairs): """pairs: list of (human verdict, gpt4 verdict). Returns (S1 agree, S1 n, S2 agree, S2 n).""" s1 = sum(1 for h, g in pairs if h == g) both = [(h, g) for h, g in pairs if h != "tie" and g != "tie"] s2 = sum(1 for h, g in both if h == g) return s1, len(pairs), s2, len(both) def main(): ap = argparse.ArgumentParser() ap.add_argument("--download", metavar="DIR") ap.add_argument("--data", metavar="DIR") ap.add_argument("--show", metavar="ROW_IDX", type=int) args = ap.parse_args() if args.download: download(args.download) return if args.show is not None: show(args.show) return if not args.data: ap.error("give --download DIR or --data DIR") human = json.load(open(os.path.join(args.data, "human.json"))) gpt4 = json.load(open(os.path.join(args.data, "gpt4_pair.json"))) assert len(human) == SPLITS["human"] and len(gpt4) == SPLITS["gpt4_pair"] g_by_key = {key(r): r for r in gpt4} assert len(g_by_key) == len(gpt4) experts_t1 = sorted((r for r in human if r["judge"].startswith("expert_") and r["turn"] == 1), key=lambda r: r["row_idx"]) step = 26 sample = experts_t1[::step][:50] print(f"expert turn-1 votes: {len(experts_t1)}; step {step}; sample {len(sample)}") print() print("| No. | row_idx | question_id | category | answer A | answer B | human | GPT-4 |") print("|---|---|---|---|---|---|---|---|") pairs, disagreements = [], [] for i, r in enumerate(sample, 1): g = g_by_key[key(r)] a, b = sorted((r["model_a"], r["model_b"])) hv, gv = verdict(r), verdict(g) label = {a: "A", b: "B", "tie": "tie"} g_label = label[gv] + ("*" if g["winner"] == "tie (inconsistent)" else "") print(f"| {i} | {r['row_idx']} | {r['question_id']} | {category(r['question_id'])} | {a} | {b} " f"| {label[hv]} | {g_label} |") pairs.append((hv, gv)) if hv != gv: disagreements.append(i) s1, n1, s2, n2 = agreement(pairs) print() print(f"sample S1 (ties included, inconsistent = tie): {s1}/{n1} = {s1/n1:.1%}") print(f"sample S2 (both non-tie): {s2}/{n2} = {s2/n2:.1%}") p_full = 886 / 1343 # turn-1 S1 agreement on all expert votes, printed below print(f"standard error of an S1 agreement near {p_full:.0%} measured on {n1} records: " f"sqrt(p(1-p)/n) = {(p_full * (1 - p_full) / n1) ** 0.5:.3f}") print(f"disagreements (No.): {disagreements}") kinds = {"human tie, GPT-4 not": 0, "GPT-4 tie, human not": 0, "opposite winners": 0} for i in disagreements: hv, gv = pairs[i - 1] if hv == "tie": kinds["human tie, GPT-4 not"] += 1 elif gv == "tie": kinds["GPT-4 tie, human not"] += 1 else: kinds["opposite winners"] += 1 print(f"disagreement kinds: {kinds}") first_opposite = [i for i in disagreements if "tie" not in pairs[i - 1]][:5] print(f"first five opposite-winner disagreements (No.): {first_opposite}") # Experiment route: every expert vote, both turns. all_pairs = [(verdict(r), verdict(g_by_key[key(r)])) for r in human if r["judge"].startswith("expert_")] s1, n1, s2, n2 = agreement(all_pairs) print() print(f"all expert votes S1: {s1}/{n1} = {s1/n1:.1%}") print(f"all expert votes S2: {s2}/{n2} = {s2/n2:.1%}") for turn in (1, 2): tp = [(verdict(r), verdict(g_by_key[key(r)])) for r in human if r["judge"].startswith("expert_") and r["turn"] == turn] s1, n1, s2, n2 = agreement(tp) print(f" turn {turn}: S1 {s1}/{n1} = {s1/n1:.1%}; S2 {s2}/{n2} = {s2/n2:.1%}") inconsistent = sum(1 for g in gpt4 if g["winner"] == "tie (inconsistent)") print(f"GPT-4 verdicts marked 'tie (inconsistent)': {inconsistent}/{len(gpt4)} = {inconsistent/len(gpt4):.1%}") if __name__ == "__main__": main()