minterp

scripts/lens.py

92 lines · View on GitHub · Linked from 1. Read

1"""
2Step 1: where in the model does the answer show? See docs/01-read.md.
3
4For every template, and for each of the three versions of its prompts,
5this records
6
7- the logit of the second name (B) minus that of the first (A), decoded from
8  the stream after the embedding and after every block, as if the model
9  stopped there (the logit lens), and
10- the final difference split into what the embedding, every head and every
11  MLP wrote directly into it (direct logit attribution).
12
13The difference is always B minus A, also where A is the answer, so that a
14part which writes the same number whoever acts can be told from a part
15whose number follows the acting name.
16
17    uv run scripts/lens.py
18    uv run scripts/lens.py --model seed1
19"""
20
21import argparse
22
23import torch
24
25from minterp import lens, tasks
26from minterp.common import load, rounded, save
27from minterp.tap import Tap
28
29parser = argparse.ArgumentParser()
30parser.add_argument("--model", default="seed0", help="seed0, seed1, or the path of a checkpoint")
31args = parser.parse_args()
32
33model, tok, meta = load(args.model)
34device = next(model.parameters()).device
35n = model.cfg.n_layers
36result = {"model": meta, "templates": []}
37VERSIONS = ("clean", "flipped", "neutral")
38pooled = {v: [] for v in VERSIONS}
39
40for batch in tasks.batches(tok, device):
41    entry = {"template": batch.template, "prompts": len(batch), "lens": {}}
42    print(batch.template)
43    print("  decoded after  " + " ".join(f"{'emb' if i == 0 else i:>6}" for i in range(n + 1)))
44    for version in VERSIONS:
45        with Tap(model) as tap, torch.no_grad():
46            logits, _ = model(getattr(batch, version))
47        final = tasks.logit_diff(logits.float(), batch.right, batch.wrong)
48        depth = torch.stack([
49            tasks.logit_diff(lens.decode(model, tap["resid", i][:, -1]).float(), batch.right, batch.wrong) for i in range(n + 1)
50        ])  # (depth, prompts), depth 0 = after the embedding
51        parts = {k: v.float() for k, v in lens.attribute(model, tap, batch.right, batch.wrong).items()}
52        total = parts["embed"] + parts["heads"].sum((1, 2)) + parts["mlp"].sum(1)
53        assert torch.allclose(total, final, atol=1e-2), "the attribution does not add up"
54        pooled[version].append(parts)
55        entry["lens"][version] = rounded(depth, 3)
56        if version == "clean":
57            # The stream's root mean square at the last position, by depth. The
58            # final norm divides by it, so the lens reads a small early stream
59            # at full size.
60            entry["size"] = rounded(torch.stack([tap["resid", i][:, -1].float().pow(2).mean(-1).sqrt().mean() for i in range(n + 1)]), 3)
61        print(f"  {version:8s} B - A " + " ".join(f"{v:+6.2f}" for v in depth.mean(1).tolist()))
62    result["templates"].append(entry)
63
64
65def summary(x):
66    """Mean and quartiles over the prompts, per part."""
67    q = torch.quantile(x, torch.tensor([0.25, 0.5, 0.75], device=x.device), dim=0)
68    return {"mean": rounded(x.mean(0), 3), "q25": rounded(q[0], 3), "median": rounded(q[1], 3), "q75": rounded(q[2], 3)}
69
70
71result["attribution"] = {}
72for version in VERSIONS:
73    cat = {k: torch.cat([p[k] for p in pooled[version]]) for k in ("embed", "heads", "mlp")}
74    result["attribution"][version] = {"prompts": len(cat["embed"]), **{k: summary(v) for k, v in cat.items()}}
75
76mean = {v: result["attribution"][v] for v in VERSIONS}
77heads = {v: torch.tensor(mean[v]["heads"]["mean"]) for v in VERSIONS}
78mlp = {v: torch.tensor(mean[v]["mlp"]["mean"]) for v in VERSIONS}
79print(f"\nwritten directly into B - A on the clean prompts, mean over {mean['clean']['prompts']} prompts")
80print("  block  " + " ".join(f"head {h}" for h in range(model.cfg.n_heads)) + "     MLP")
81for i in range(n):
82    print(f"  {i + 1:5d}  " + " ".join(f"{v:+6.2f}" for v in heads["clean"][i].tolist()) + f"  {mlp['clean'][i].item():+6.2f}")
83ranked = [(abs(heads["clean"][i, h].item()), f"block {i + 1} head {h}", ("heads", i, h)) for i in range(n) for h in range(model.cfg.n_heads)]
84ranked += [(abs(mlp["clean"][i].item()), f"block {i + 1} MLP", ("mlp", i)) for i in range(n)]
85ranked.sort(reverse=True)
86print("\n  the largest parts, and what they write in each version")
87print(f"  {'':16s} {'clean':>7} {'flipped':>8} {'neutral':>8}")
88for _, name, key in ranked[:8]:
89    get = lambda v: (heads[v][key[1], key[2]] if key[0] == "heads" else mlp[v][key[1]]).item()
90    print(f"  {name:16s} {get('clean'):+7.2f} {get('flipped'):+8.2f} {get('neutral'):+8.2f}")
91
92save(f"runs/read/lens-{meta['name']}.json", result)