minterp

scripts/heads.py

169 lines · View on GitHub · Linked from 8. Limits

1"""
2Step 8, the open model part by part: where the heads that step 3's patch
3found look, what they write, and which of them the head at the acting name
4works through. See docs/08-limits.md.
5
6The heads are not chosen by hand. They are every head whose share of the
7gap between the clean and the flipped prompt is 0.05 or more in either
8direction, in the record of scripts/patch.py. For each of them, per
9sentence pattern and version of the prompt, this records
10
11attention   from the position where it acts (the acting name if step 3
12            found its share there, the last position otherwise), how much
13            it reads from the first token, the first name, the second name,
14            the full stop that ends the first sentence, the acting name,
15            the token after it and the last position; and the position it
16            reads most from among all the others
17writes      for a head read at the last position, what it adds to the
18            logit of A, of B and of C, each against the mean of the nine
19            other names
20by kind     the same on clean and flipped prompts for the name that acts
21            and the name that is due, each against the uninvolved names of
22            its own kind, where a kind is the names the model follows with
23            " She" or with " He" (step 6)
24holds       for a head that acts at the acting name and every later head:
25            its share alone, and what is left of it when the later head is
26            held at the flipped run's own output (step 4)
27
28The first two and the last are measures of step 4 (scripts/circuit.py), on
29another choice of heads: step 4 takes the heads with a positive share or a
30large direct write. tests/test_limits.py checks that on the story models
31the two scripts give the same numbers for the heads they share.
32
33    uv run scripts/heads.py                  # the open model: under a minute
34    uv run scripts/heads.py --model seed0    # a story model, as a check against step 4
35"""
36
37import argparse
38import json
39
40import torch
41
42from minterp import attention, patch, tasks
43from minterp.common import eot, load, save
44from minterp.tap import Tap, head_writes
45
46parser = argparse.ArgumentParser()
47parser.add_argument("--model", default="qwen", help="qwen, seed0, seed1 or seed2: scripts/patch.py must have run on it, with its parts")
48args = parser.parse_args()
49
50model, tok, meta = load(args.model)
51device = next(model.parameters()).device
52n, H = model.cfg.n_layers, model.cfg.n_heads
53VERSIONS = ("clean", "flipped", "neutral")
54PLACES = ("first token", "A", "B", "stop", "S", "S+1", "end")
55
56with open(f"runs/patch/patch-{meta['name']}.json") as f:
57    step3 = json.load(f)["templates"]
58carry = {s: torch.tensor([t["parts"]["carry"][s] for t in step3]).mean(0)[:, :H] for s in ("all", "S")}  # (blocks, heads)
59heads = [{
60    "block": b + 1, "head": h, "carry": round(carry["all"][b, h].item(), 3),
61    # A head acts at the acting name if step 3 found its share there; all others are read at the last position.
62    "acts at": "S" if carry["S"][b, h].abs() >= 0.05 else "end",
63} for b in range(n) for h in range(H) if carry["all"][b, h].abs() >= 0.05]
64name_of = lambda x: f"block {x['block']} head {x['head']}"
65key = lambda x: ("heads", x["block"])
66people = tasks.names(tok)
67ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
68stop_id = tok.encode_ordinary(".")[0]
69
70# kinds, as scripts/open.py finds them: after a sentence about one name, does the model go on with " She" or " He"?
71she_id, he_id = (tok.encode_ordinary(w)[0] for w in (" She", " He"))
72with torch.no_grad():
73    after = model(torch.tensor([[eot(tok)] + tok.encode_ordinary(f"One day, {p} went to the park.") for p in people], device=device))[0][:, -1].float()
74she = after[:, she_id] > after[:, he_id]
75
76result = {"model": meta, "heads": heads, "versions": VERSIONS, "places": PLACES,
77          "names that go on with She": [p for p, g in zip(people, she.tolist()) if g], "templates": []}
78print("the heads:", ", ".join(f"{name_of(x)} (at {x['acts at']})" for x in heads))
79
80for batch in tasks.batches(tok, device, people=people):
81    p = batch.pos
82    entry = {"template": batch.template, "attention": {}, "elsewhere": {}, "writes": {}, "by kind": {}, "holds": []}
83    # The full stop that ends the first sentence: the first one after the second name.
84    stop = next(i for i in range(p["B"] + 1, p["S"]) if batch.clean[0, i].item() == stop_id)
85    place = {"first token": 0, "A": p["A"], "B": p["B"], "stop": stop, "S": p["S"], "S+1": p["S"] + 1, "end": p["end"]}
86    index = {name: i for i, name in enumerate(people)}
87    rows = torch.arange(len(batch), device=device)
88    who = {k: torch.tensor([index[t[i]] for t in batch.names], device=device) for i, k in enumerate(("A", "B", "C"))}
89    uninvolved = torch.ones(len(batch), len(people), dtype=torch.bool, device=device)
90    for k in who:
91        uninvolved[rows, who[k]] = False
92
93    for version in VERSIONS:
94        with torch.no_grad(), Tap(model, keep=("resid", "heads")) as tap:
95            model(getattr(batch, version))
96        final = tap["resid", n][:, -1].float()
97        size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
98        weights = {}
99        for x in heads:
100            b = x["block"]
101            if b not in weights:
102                with torch.no_grad():
103                    weights = {b: attention.weights(model, model.layers[b - 1].attn, model.layers[b - 1].attn_norm(tap["resid", b - 1])).float()}
104            row = weights[b][:, x["head"], p[x["acts at"]]]  # (prompts, seq): where it reads from
105            entry["attention"].setdefault(name_of(x), {})[version] = {k: row[:, i].mean().item() for k, i in place.items()}
106            # The most-read position that is none of the seven places.
107            others = [i for i in range(p[x["acts at"]] + 1) if i not in place.values()]
108            most = max(others, key=lambda i: row[:, i].mean().item())
109            entry["elsewhere"].setdefault(name_of(x), {})[version] = {"token": batch.labels[most], "weight": row[:, most].mean().item()}
110            if x["acts at"] == "end":
111                write = head_writes(model, tap, b)[:, -1, x["head"]].float()
112                logits = (write * model.norm.weight.float() * size) @ model.lm_head.weight[ids].float().T  # (prompts, names)
113                rest = (logits * uninvolved).sum(1) / uninvolved.sum(1)
114                entry["writes"].setdefault(name_of(x), {})[version] = {k: (logits[rows, who[k]] - rest).mean().item() for k in who}
115                if version != "neutral":
116                    # Step 6's question: against the uninvolved names of the name's own kind, by that kind.
117                    acts, due = (who["A"], who["B"]) if version == "clean" else (who["B"], who["A"])
118                    split = {}
119                    for role, target in (("acts", acts), ("due", due)):
120                        own = uninvolved & (she[None, :] == she[target][:, None])
121                        against_own = logits[rows, target] - (logits * own).sum(1) / own.sum(1).clamp(min=1)
122                        usable = own.sum(1) > 0
123                        split[role] = {label: against_own[mask & usable].mean().item() for label, mask in (("She", she[target]), ("He", ~she[target]))}
124                    entry["by kind"].setdefault(name_of(x), {})[version] = split
125        del tap, weights
126
127    # holds: the sender's clean output goes into the flipped run; the later head keeps what it had there.
128    ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
129    clean, clean_logits = patch.record(model, batch.clean, keep=("heads",))
130    flipped, flipped_logits = patch.record(model, batch.flipped, keep=("heads",))
131    base, goal = ld(flipped_logits), ld(clean_logits)
132    for s in heads:
133        if s["acts at"] != "S":
134            continue
135        send = {key(s): patch.swap(clean[key(s)], head=s["head"])}
136        alone = patch.share(ld(patch.run(model, batch.flipped, send)), base, goal)
137        for r in heads:
138            if r["block"] <= s["block"]:
139                continue
140            edits = dict(send)
141            edits[key(r)] = patch.swap(flipped[key(r)], head=r["head"])  # another block, so another key
142            entry["holds"].append({"sender": name_of(s), "receiver": name_of(r), "alone": alone,
143                                   "held": patch.share(ld(patch.run(model, batch.flipped, edits)), base, goal)})
144    del clean, flipped
145    result["templates"].append(entry)
146
147T = result["templates"]
148mean = lambda pick: sum(pick(t) for t in T) / len(T)
149print(f"\ngoes on with She after {', '.join(result['names that go on with She'])}")
150print("\nwhere each head reads from, at the position where it acts (clean | flipped | neutral): " + ", ".join(PLACES) + "; then the most-read other token on clean prompts, by pattern")
151for x in heads:
152    print(f"  {name_of(x):17s} {x['carry']:6.2f} at {x['acts at']:3s}  " + "  |  ".join(
153        " ".join(f"{mean(lambda t, v=v, k=k: t['attention'][name_of(x)][v][k]):.2f}" for k in PLACES) for v in VERSIONS)
154        + "   " + ", ".join(f"{t['elsewhere'][name_of(x)]['clean']['token'].strip()} {t['elsewhere'][name_of(x)]['clean']['weight']:.2f}" for t in T))
155print("\nwhat the heads at the last position add to the logit of A, B, C against the other names (clean | flipped | neutral)")
156for x in heads:
157    if x["acts at"] == "end":
158        print(f"  {name_of(x):17s}  " + "  |  ".join(
159            " ".join(f"{mean(lambda t, v=v, k=k: t['writes'][name_of(x)][v][k]):+.2f}" for k in ("A", "B", "C")) for v in VERSIONS))
160print("\nthe same against the uninvolved names of the name's own kind: the name that acts if She, if He; the name that is due if She, if He (clean | flipped)")
161for x in heads:
162    if x["acts at"] == "end":
163        print(f"  {name_of(x):17s}  " + "  |  ".join(
164            " ".join(f"{mean(lambda t, v=v, r=r, g=g: t['by kind'][name_of(x)][v][r][g]):+.2f}" for r in ("acts", "due") for g in ("She", "He")) for v in ("clean", "flipped")))
165print("\nholds: the share of a head at the acting name alone, and what is left when a later head is held at the flipped run's own output")
166for i, h in enumerate(T[0]["holds"]):
167    print(f"  {h['sender']} alone {mean(lambda t: t['holds'][i]['alone']):.2f}; with {h['receiver']:17s} held {mean(lambda t: t['holds'][i]['held']):.2f}")
168
169save(f"runs/open/heads-{meta['name']}.json", json.loads(json.dumps(result), parse_float=lambda s: round(float(s), 4)))