scripts/circuit.py
224 lines · View on GitHub · Linked from 4. Circuit
1"""
2Step 4: what do the parts found in step 3 do, and which works through which? See docs/04-circuit.md.
3
4The heads are not chosen by hand. They are every head that step 3 found to
5carry at least 0.05 of the gap between the clean and the flipped prompt,
6and every head that step 1 found to write at least 0.3 logits into the
7answer in some version. For each of them this records
8
9attention from the position where it acts (the acting name or the last
10 position, as step 3 found), how much it reads from the first
11 name, the second name, the full stop that ends the first
12 sentence, the acting name and the first token
13writes at the last position, what it adds to the logit of A, of B and
14 of C, each against the mean of the nine other names
15sources the same writes split by the position they were read from
16holds for every earlier head and later head of the set: how much of
17 the earlier one's share is left when the later one is held at
18 the target run's own output, and how much comes along when the
19 later head alone is given what it wrote in the patched run
20steers for the heads that read from the names: where they look in the
21 flipped run when one earlier head is given its clean output
22sets the share carried by all heads of the set moved together, by
23 all other heads moved together, and by random sets of as many
24
25 uv run scripts/circuit.py
26 uv run scripts/circuit.py --model seed1
27"""
28
29import argparse
30import json
31
32import torch
33
34from minterp import attention, patch, tasks
35from minterp.common import load, rounded, save
36from minterp.tap import Tap, head_writes
37
38parser = argparse.ArgumentParser()
39parser.add_argument("--model", default="seed0", help="seed0, seed1 or seed2: the steps before must have run on it")
40parser.add_argument("--random-sets", type=int, default=200)
41args = parser.parse_args()
42
43model, tok, meta = load(args.model)
44device = next(model.parameters()).device
45n, H = model.cfg.n_layers, model.cfg.n_heads
46VERSIONS = ("clean", "flipped", "neutral")
47
48
49with open(f"runs/patch/patch-{meta['name']}.json") as f:
50 step3 = json.load(f)["templates"]
51with open(f"runs/read/lens-{meta['name']}.json") as f:
52 step1 = json.load(f)["attribution"]
53carry = {s: torch.tensor([t["parts"]["carry"][s] for t in step3]).mean(0)[:, :H] for s in ("all", "last", "S")}
54wrote = torch.stack([torch.tensor(step1[v]["heads"]["mean"]) for v in VERSIONS]).abs().max(0).values
55chosen = [(b, h) for b in range(n) for h in range(H) if carry["all"][b, h] >= 0.05 or wrote[b, h] >= 0.3]
56heads = [{
57 "block": b + 1, "head": h, "carry": round(carry["all"][b, h].item(), 3),
58
59 "acts at": "S" if carry["S"][b, h] >= 0.05 else "end",
60 "writes": [round(step1[v]["heads"]["mean"][b][h], 3) for v in VERSIONS],
61} for b, h in chosen]
62name_of = lambda x: f"block {x['block']} head {x['head']}"
63people = tasks.names(tok)
64ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
65stop_id = tok.encode_ordinary(".")[0]
66PLACES = ("A", "B", "stop", "S", "first token")
67result = {"model": meta, "heads": heads, "versions": VERSIONS, "places": PLACES, "templates": []}
68print("the heads:", ", ".join(f"{name_of(x)} (at {x['acts at']})" for x in heads))
69
70key = lambda x: ("heads", x["block"])
71readers = [x for x in heads if x["acts at"] == "end"]
72
73
74def as_tap(store):
75 return type("Store", (), {"__getitem__": lambda self, k: store[k]})()
76
77
78def run_tapped(tokens, edits):
79 with Tap(model, edit=edits) as tap, torch.no_grad():
80 logits, _ = model(tokens)
81 return tap, logits
82
83
84generator = torch.Generator().manual_seed(0)
85random_sets = [torch.randperm(n * H, generator=generator)[: len(heads)].tolist() for _ in range(args.random_sets)]
86result["random sets"] = [[f"block {i // H + 1} head {i % H}" for i in sorted(s)] for s in random_sets]
87
88for batch in tasks.batches(tok, device):
89 p = batch.pos
90 ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
91 store, value = {}, {}
92 for version in VERSIONS:
93 store[version], value[version] = patch.record(model, getattr(batch, version))
94 base, goal = ld(value["flipped"]), ld(value["clean"])
95 entry = {"template": batch.template, "attention": {}, "writes": {}, "sources": {}}
96
97 stop = next(i for i in range(p["B"] + 1, p["S"]) if batch.clean[0, i].item() == stop_id)
98 place = {"A": p["A"], "B": p["B"], "stop": stop, "S": p["S"], "first token": 0}
99
100 index = {name: i for i, name in enumerate(people)}
101 rows = torch.arange(len(batch), device=device)
102 who = {k: torch.tensor([index[t[i]] for t in batch.names], device=device) for i, k in enumerate(("A", "B", "C"))}
103 uninvolved = torch.ones(len(batch), len(people), dtype=torch.bool, device=device)
104 for k in who:
105 uninvolved[rows, who[k]] = False
106
107 for version in VERSIONS:
108 tap = as_tap(store[version])
109 w = attention.patterns(model, tap)
110 final = store[version]["resid", n][:, -1]
111 size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
112 for x in heads:
113 row = w[:, x["block"] - 1, x["head"], p[x["acts at"]]]
114 entry["attention"].setdefault(name_of(x), {})[version] = {k: row[:, i].mean().item() for k, i in place.items()}
115 if x["acts at"] == "end":
116 write = head_writes(model, tap, x["block"])[:, -1, x["head"]]
117 logits = ((write * model.norm.weight * size) @ model.lm_head.weight[ids].T).float()
118 rest = (logits * uninvolved).sum(1) / uninvolved.sum(1)
119 entry["writes"].setdefault(name_of(x), {})[version] = {k: (logits[rows, who[k]] - rest).mean().item() for k in who}
120
121 attn = model.layers[x["block"] - 1].attn
122 v = attention.values(model, attn, model.layers[x["block"] - 1].attn_norm(store[version]["resid", x["block"] - 1]))
123 hd = attn.head_dim
124 O = attn.o_proj.weight[:, x["head"] * hd:(x["head"] + 1) * hd]
125 moved = (row[:, :, None] * v[:, x["head"]]) @ O.T
126 by_pos = ((moved * model.norm.weight * size[:, None]) @ model.lm_head.weight[ids].T).float()
127 by_pos = by_pos - (by_pos * uninvolved[:, None]).sum(2, keepdim=True) / uninvolved.sum(1)[:, None, None]
128 split = {}
129 for k in ("A", "B", "stop", "S"):
130 split[k] = {m: by_pos[rows, place[k], who[m]].mean().item() for m in ("A", "B")}
131 total = {m: by_pos[rows, :, who[m]].sum(1).mean().item() for m in ("A", "B")}
132 split["elsewhere"] = {m: total[m] - sum(split[k][m] for k in ("A", "B", "stop", "S")) for m in ("A", "B")}
133 entry["sources"].setdefault(name_of(x), {})[version] = split
134
135
136 entry["holds"] = []
137 for s in heads:
138 send = {key(s): patch.swap(store["clean"][key(s)], head=s["head"])}
139 sent, logits = run_tapped(batch.flipped, send)
140 alone = patch.share(ld(logits), base, goal)
141 for r in heads:
142 if r["block"] <= s["block"]:
143 continue
144 edits = dict(send)
145 edits[key(r)] = patch.swap(store["flipped"][key(r)], head=r["head"])
146 held = patch.share(ld(patch.run(model, batch.flipped, edits)), base, goal)
147
148 relay = {key(r): patch.swap(sent[key(r)], head=r["head"])}
149 relayed = patch.share(ld(patch.run(model, batch.flipped, relay)), base, goal)
150 entry["holds"].append({"sender": name_of(s), "receiver": name_of(r), "alone": alone, "held": held, "relayed": relayed})
151
152
153 entry["steers"] = []
154 plain = attention.patterns(model, as_tap(store["flipped"]))
155 clean_w = attention.patterns(model, as_tap(store["clean"]))
156 for r in readers:
157 look = lambda w: {k: w[:, r["block"] - 1, r["head"], p["end"], place[k]].mean().item() for k in ("A", "B", "stop", "S")}
158 for s in heads:
159 if s["block"] >= r["block"]:
160 continue
161 tap, _ = run_tapped(batch.flipped, {key(s): patch.swap(store["clean"][key(s)], head=s["head"])})
162
163 final = tap["resid", n][:, -1]
164 size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
165 write = head_writes(model, tap, r["block"])[:, -1, r["head"]]
166 logits = ((write * model.norm.weight * size) @ model.lm_head.weight[ids].T).float()
167 rest = (logits * uninvolved).sum(1) / uninvolved.sum(1)
168 entry["steers"].append({"reader": name_of(r), "sender": name_of(s), "flipped": look(plain), "clean": look(clean_w),
169 "patched": look(attention.patterns(model, tap)),
170 "patched writes": {k: (logits[rows, who[k]] - rest).mean().item() for k in ("A", "B")}})
171
172
173 def together(flat):
174 edits = {}
175 for i in flat:
176 b, h = divmod(i, H)
177 k = ("heads", b + 1)
178 edit = patch.swap(store["clean"][k], head=h)
179 edits[k] = (lambda x, first=edits[k], second=edit: second(first(x))) if k in edits else edit
180 return patch.share(ld(patch.run(model, batch.flipped, edits)), base, goal)
181
182 mine = [(x["block"] - 1) * H + x["head"] for x in heads]
183 carriers = [(x["block"] - 1) * H + x["head"] for x in heads if x["carry"] >= 0.05]
184 entry["sets"] = {
185 "the heads": together(mine), "the heads that carry": together(carriers),
186 "all other heads": together([i for i in range(n * H) if i not in mine]), "all heads": together(list(range(n * H))),
187 "random": [together(s) for s in random_sets],
188 }
189 result["templates"].append(entry)
190 print(batch.template, {k: round(v, 2) for k, v in entry["sets"].items() if k != "random"})
191
192mean = lambda values: sum(values) / len(values)
193T = result["templates"]
194print("\nwhere each head reads from, at the position where it acts (clean | flipped | neutral)")
195for x in heads:
196 a = lambda v, k: mean([t["attention"][name_of(x)][v][k] for t in T])
197 print(f" {name_of(x):15s} at {x['acts at']:3s} " + " | ".join(" ".join(f"{k} {a(v, k):.2f}" for k in ("A", "B", "stop", "S")) for v in VERSIONS))
198print("\nwhat each head at the last position adds to a name's logit, against the nine names that are none of A, B and C (clean | flipped | neutral)")
199for x in readers:
200 wv = lambda v, k: mean([t["writes"][name_of(x)][v][k] for t in T])
201 print(f" {name_of(x):15s} " + " | ".join(" ".join(f"{k} {wv(v, k):+.2f}" for k in ("A", "B", "C")) for v in VERSIONS))
202print("\nwhere the writes of the heads at the last position come from, on clean and flipped prompts (to A, to B)")
203for x in readers:
204 sv = lambda v, k, m: mean([t["sources"][name_of(x)][v][k][m] for t in T])
205 print(f" {name_of(x):15s} " + " | ".join(" ".join(f"{k} {sv(v, k, 'A'):+.2f},{sv(v, k, 'B'):+.2f}" for k in ("A", "B", "stop", "S", "elsewhere")) for v in ("clean", "flipped")))
206print("\nholds: an earlier head's share alone, with a later head held, and relayed by the later head alone (only where held changes by 0.03 or more)")
207for i in range(len(T[0]["holds"])):
208 alone, held, relayed = (mean([t["holds"][i][k] for t in T]) for k in ("alone", "held", "relayed"))
209 if abs(alone - held) >= 0.03:
210 print(f" {T[0]['holds'][i]['sender']:15s} {alone:5.2f} -> {held:5.2f} with {T[0]['holds'][i]['receiver']} held; relayed {relayed:5.2f}")
211print("\nsteers: where a head at the last position looks in the flipped run (A, B), with one earlier head from the clean run (only shifts of 0.05 or more)")
212for i in range(len(T[0]["steers"])):
213 g = lambda which, k: mean([t["steers"][i][which][k] for t in T])
214 if abs(g("patched", "A") - g("flipped", "A")) + abs(g("patched", "B") - g("flipped", "B")) >= 0.05:
215 s = T[0]["steers"][i]
216 print(f" {s['reader']:15s} flipped A {g('flipped', 'A'):.2f} B {g('flipped', 'B'):.2f} | with {s['sender']} clean: A {g('patched', 'A'):.2f} B {g('patched', 'B'):.2f} | clean run A {g('clean', 'A'):.2f} B {g('clean', 'B'):.2f}")
217sets = {k: mean([t["sets"][k] for t in T]) for k in T[0]["sets"] if k != "random"}
218rand = torch.tensor([t["sets"]["random"] for t in T]).mean(0)
219print(f"\nsets of heads moved together: {', '.join(f'{k} {v:.2f}' for k, v in sets.items())}")
220print(f" {len(random_sets)} random sets of {len(heads)} heads: median {rand.median():.2f}, 95th percentile {rand.quantile(0.95):.2f}, largest {rand.max():.2f}")
221
222for t in T:
223 t["sets"]["random"] = rounded(torch.tensor(t["sets"]["random"]), 3)
224save(f"runs/circuit/circuit-{meta['name']}.json", json.loads(json.dumps(result), parse_float=lambda s: round(float(s), 4)))