scripts/lens.py
92 lines · View on GitHub · Linked from 1. Read
1"""
2Step 1: where in the model does the answer show? See docs/01-read.md.
3
4For every template, and for each of the three versions of its prompts,
5this records
6
7- the logit of the second name (B) minus that of the first (A), decoded from
8 the stream after the embedding and after every block, as if the model
9 stopped there (the logit lens), and
10- the final difference split into what the embedding, every head and every
11 MLP wrote directly into it (direct logit attribution).
12
13The difference is always B minus A, also where A is the answer, so that a
14part which writes the same number whoever acts can be told from a part
15whose number follows the acting name.
16
17 uv run scripts/lens.py
18 uv run scripts/lens.py --model seed1
19"""
20
21import argparse
22
23import torch
24
25from minterp import lens, tasks
26from minterp.common import load, rounded, save
27from minterp.tap import Tap
28
29parser = argparse.ArgumentParser()
30parser.add_argument("--model", default="seed0", help="seed0, seed1, or the path of a checkpoint")
31args = parser.parse_args()
32
33model, tok, meta = load(args.model)
34device = next(model.parameters()).device
35n = model.cfg.n_layers
36result = {"model": meta, "templates": []}
37VERSIONS = ("clean", "flipped", "neutral")
38pooled = {v: [] for v in VERSIONS}
39
40for batch in tasks.batches(tok, device):
41 entry = {"template": batch.template, "prompts": len(batch), "lens": {}}
42 print(batch.template)
43 print(" decoded after " + " ".join(f"{'emb' if i == 0 else i:>6}" for i in range(n + 1)))
44 for version in VERSIONS:
45 with Tap(model) as tap, torch.no_grad():
46 logits, _ = model(getattr(batch, version))
47 final = tasks.logit_diff(logits.float(), batch.right, batch.wrong)
48 depth = torch.stack([
49 tasks.logit_diff(lens.decode(model, tap["resid", i][:, -1]).float(), batch.right, batch.wrong) for i in range(n + 1)
50 ])
51 parts = {k: v.float() for k, v in lens.attribute(model, tap, batch.right, batch.wrong).items()}
52 total = parts["embed"] + parts["heads"].sum((1, 2)) + parts["mlp"].sum(1)
53 assert torch.allclose(total, final, atol=1e-2), "the attribution does not add up"
54 pooled[version].append(parts)
55 entry["lens"][version] = rounded(depth, 3)
56 if version == "clean":
57
58
59
60 entry["size"] = rounded(torch.stack([tap["resid", i][:, -1].float().pow(2).mean(-1).sqrt().mean() for i in range(n + 1)]), 3)
61 print(f" {version:8s} B - A " + " ".join(f"{v:+6.2f}" for v in depth.mean(1).tolist()))
62 result["templates"].append(entry)
63
64
65def summary(x):
66 """Mean and quartiles over the prompts, per part."""
67 q = torch.quantile(x, torch.tensor([0.25, 0.5, 0.75], device=x.device), dim=0)
68 return {"mean": rounded(x.mean(0), 3), "q25": rounded(q[0], 3), "median": rounded(q[1], 3), "q75": rounded(q[2], 3)}
69
70
71result["attribution"] = {}
72for version in VERSIONS:
73 cat = {k: torch.cat([p[k] for p in pooled[version]]) for k in ("embed", "heads", "mlp")}
74 result["attribution"][version] = {"prompts": len(cat["embed"]), **{k: summary(v) for k, v in cat.items()}}
75
76mean = {v: result["attribution"][v] for v in VERSIONS}
77heads = {v: torch.tensor(mean[v]["heads"]["mean"]) for v in VERSIONS}
78mlp = {v: torch.tensor(mean[v]["mlp"]["mean"]) for v in VERSIONS}
79print(f"\nwritten directly into B - A on the clean prompts, mean over {mean['clean']['prompts']} prompts")
80print(" block " + " ".join(f"head {h}" for h in range(model.cfg.n_heads)) + " MLP")
81for i in range(n):
82 print(f" {i + 1:5d} " + " ".join(f"{v:+6.2f}" for v in heads["clean"][i].tolist()) + f" {mlp['clean'][i].item():+6.2f}")
83ranked = [(abs(heads["clean"][i, h].item()), f"block {i + 1} head {h}", ("heads", i, h)) for i in range(n) for h in range(model.cfg.n_heads)]
84ranked += [(abs(mlp["clean"][i].item()), f"block {i + 1} MLP", ("mlp", i)) for i in range(n)]
85ranked.sort(reverse=True)
86print("\n the largest parts, and what they write in each version")
87print(f" {'':16s} {'clean':>7} {'flipped':>8} {'neutral':>8}")
88for _, name, key in ranked[:8]:
89 get = lambda v: (heads[v][key[1], key[2]] if key[0] == "heads" else mlp[v][key[1]]).item()
90 print(f" {name:16s} {get('clean'):+7.2f} {get('flipped'):+8.2f} {get('neutral'):+8.2f}")
91
92save(f"runs/read/lens-{meta['name']}.json", result)