minterp

scripts/circuit.py

224 lines · View on GitHub · Linked from 4. Circuit

1"""
2Step 4: what do the parts found in step 3 do, and which works through which? See docs/04-circuit.md.
3
4The heads are not chosen by hand. They are every head that step 3 found to
5carry at least 0.05 of the gap between the clean and the flipped prompt,
6and every head that step 1 found to write at least 0.3 logits into the
7answer in some version. For each of them this records
8
9attention   from the position where it acts (the acting name or the last
10            position, as step 3 found), how much it reads from the first
11            name, the second name, the full stop that ends the first
12            sentence, the acting name and the first token
13writes      at the last position, what it adds to the logit of A, of B and
14            of C, each against the mean of the nine other names
15sources     the same writes split by the position they were read from
16holds       for every earlier head and later head of the set: how much of
17            the earlier one's share is left when the later one is held at
18            the target run's own output, and how much comes along when the
19            later head alone is given what it wrote in the patched run
20steers      for the heads that read from the names: where they look in the
21            flipped run when one earlier head is given its clean output
22sets        the share carried by all heads of the set moved together, by
23            all other heads moved together, and by random sets of as many
24
25    uv run scripts/circuit.py
26    uv run scripts/circuit.py --model seed1
27"""
28
29import argparse
30import json
31
32import torch
33
34from minterp import attention, patch, tasks
35from minterp.common import load, rounded, save
36from minterp.tap import Tap, head_writes
37
38parser = argparse.ArgumentParser()
39parser.add_argument("--model", default="seed0", help="seed0, seed1 or seed2: the steps before must have run on it")
40parser.add_argument("--random-sets", type=int, default=200)
41args = parser.parse_args()
42
43model, tok, meta = load(args.model)
44device = next(model.parameters()).device
45n, H = model.cfg.n_layers, model.cfg.n_heads
46VERSIONS = ("clean", "flipped", "neutral")
47
48# The heads, from the records of steps 1 and 3.
49with open(f"runs/patch/patch-{meta['name']}.json") as f:
50    step3 = json.load(f)["templates"]
51with open(f"runs/read/lens-{meta['name']}.json") as f:
52    step1 = json.load(f)["attribution"]
53carry = {s: torch.tensor([t["parts"]["carry"][s] for t in step3]).mean(0)[:, :H] for s in ("all", "last", "S")}  # (blocks, heads)
54wrote = torch.stack([torch.tensor(step1[v]["heads"]["mean"]) for v in VERSIONS]).abs().max(0).values
55chosen = [(b, h) for b in range(n) for h in range(H) if carry["all"][b, h] >= 0.05 or wrote[b, h] >= 0.3]
56heads = [{
57    "block": b + 1, "head": h, "carry": round(carry["all"][b, h].item(), 3),
58    # A head acts at the acting name if step 3 found its share there; all others are read at the last position.
59    "acts at": "S" if carry["S"][b, h] >= 0.05 else "end",
60    "writes": [round(step1[v]["heads"]["mean"][b][h], 3) for v in VERSIONS],
61} for b, h in chosen]
62name_of = lambda x: f"block {x['block']} head {x['head']}"
63people = tasks.names(tok)
64ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
65stop_id = tok.encode_ordinary(".")[0]
66PLACES = ("A", "B", "stop", "S", "first token")
67result = {"model": meta, "heads": heads, "versions": VERSIONS, "places": PLACES, "templates": []}
68print("the heads:", ", ".join(f"{name_of(x)} (at {x['acts at']})" for x in heads))
69
70key = lambda x: ("heads", x["block"])
71readers = [x for x in heads if x["acts at"] == "end"]
72
73
74def as_tap(store):
75    return type("Store", (), {"__getitem__": lambda self, k: store[k]})()
76
77
78def run_tapped(tokens, edits):
79    with Tap(model, edit=edits) as tap, torch.no_grad():
80        logits, _ = model(tokens)
81    return tap, logits
82
83
84generator = torch.Generator().manual_seed(0)
85random_sets = [torch.randperm(n * H, generator=generator)[: len(heads)].tolist() for _ in range(args.random_sets)]
86result["random sets"] = [[f"block {i // H + 1} head {i % H}" for i in sorted(s)] for s in random_sets]
87
88for batch in tasks.batches(tok, device):
89    p = batch.pos
90    ld = lambda logits: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
91    store, value = {}, {}
92    for version in VERSIONS:
93        store[version], value[version] = patch.record(model, getattr(batch, version))
94    base, goal = ld(value["flipped"]), ld(value["clean"])
95    entry = {"template": batch.template, "attention": {}, "writes": {}, "sources": {}}
96    # The full stop that ends the first sentence: the first one after the second name.
97    stop = next(i for i in range(p["B"] + 1, p["S"]) if batch.clean[0, i].item() == stop_id)
98    place = {"A": p["A"], "B": p["B"], "stop": stop, "S": p["S"], "first token": 0}
99
100    index = {name: i for i, name in enumerate(people)}
101    rows = torch.arange(len(batch), device=device)
102    who = {k: torch.tensor([index[t[i]] for t in batch.names], device=device) for i, k in enumerate(("A", "B", "C"))}
103    uninvolved = torch.ones(len(batch), len(people), dtype=torch.bool, device=device)
104    for k in who:
105        uninvolved[rows, who[k]] = False
106
107    for version in VERSIONS:
108        tap = as_tap(store[version])
109        w = attention.patterns(model, tap)  # (prompts, blocks, heads, seq, seq)
110        final = store[version]["resid", n][:, -1]
111        size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
112        for x in heads:
113            row = w[:, x["block"] - 1, x["head"], p[x["acts at"]]]  # (prompts, seq): where it reads from
114            entry["attention"].setdefault(name_of(x), {})[version] = {k: row[:, i].mean().item() for k, i in place.items()}
115            if x["acts at"] == "end":
116                write = head_writes(model, tap, x["block"])[:, -1, x["head"]]
117                logits = ((write * model.norm.weight * size) @ model.lm_head.weight[ids].T).float()  # (prompts, names)
118                rest = (logits * uninvolved).sum(1) / uninvolved.sum(1)
119                entry["writes"].setdefault(name_of(x), {})[version] = {k: (logits[rows, who[k]] - rest).mean().item() for k in who}
120                # The same write, split by where it was read: the weight on a position times that position's value.
121                attn = model.layers[x["block"] - 1].attn
122                v = attention.values(model, attn, model.layers[x["block"] - 1].attn_norm(store[version]["resid", x["block"] - 1]))
123                hd = attn.head_dim
124                O = attn.o_proj.weight[:, x["head"] * hd:(x["head"] + 1) * hd]  # (dim, head_dim)
125                moved = (row[:, :, None] * v[:, x["head"]]) @ O.T  # (prompts, seq, dim): what came from each position
126                by_pos = ((moved * model.norm.weight * size[:, None]) @ model.lm_head.weight[ids].T).float()  # (prompts, seq, names)
127                by_pos = by_pos - (by_pos * uninvolved[:, None]).sum(2, keepdim=True) / uninvolved.sum(1)[:, None, None]
128                split = {}
129                for k in ("A", "B", "stop", "S"):
130                    split[k] = {m: by_pos[rows, place[k], who[m]].mean().item() for m in ("A", "B")}
131                total = {m: by_pos[rows, :, who[m]].sum(1).mean().item() for m in ("A", "B")}
132                split["elsewhere"] = {m: total[m] - sum(split[k][m] for k in ("A", "B", "stop", "S")) for m in ("A", "B")}
133                entry["sources"].setdefault(name_of(x), {})[version] = split
134
135    # holds: the sender's clean output goes into the flipped run; the receiver keeps what it had there.
136    entry["holds"] = []
137    for s in heads:
138        send = {key(s): patch.swap(store["clean"][key(s)], head=s["head"])}
139        sent, logits = run_tapped(batch.flipped, send)
140        alone = patch.share(ld(logits), base, goal)
141        for r in heads:
142            if r["block"] <= s["block"]:
143                continue
144            edits = dict(send)
145            edits[key(r)] = patch.swap(store["flipped"][key(r)], head=r["head"])  # another block, so another key
146            held = patch.share(ld(patch.run(model, batch.flipped, edits)), base, goal)
147            # The converse: the earlier head is left alone, and the later head gets what it wrote when the earlier one was patched.
148            relay = {key(r): patch.swap(sent[key(r)], head=r["head"])}
149            relayed = patch.share(ld(patch.run(model, batch.flipped, relay)), base, goal)
150            entry["holds"].append({"sender": name_of(s), "receiver": name_of(r), "alone": alone, "held": held, "relayed": relayed})
151
152    # steers: where a reader of names looks in the flipped run, plain and with one earlier head given its clean output
153    entry["steers"] = []
154    plain = attention.patterns(model, as_tap(store["flipped"]))
155    clean_w = attention.patterns(model, as_tap(store["clean"]))
156    for r in readers:
157        look = lambda w: {k: w[:, r["block"] - 1, r["head"], p["end"], place[k]].mean().item() for k in ("A", "B", "stop", "S")}
158        for s in heads:
159            if s["block"] >= r["block"]:
160                continue
161            tap, _ = run_tapped(batch.flipped, {key(s): patch.swap(store["clean"][key(s)], head=s["head"])})
162            # What the reader then adds to the two names, on the scale of the patched run's final stream.
163            final = tap["resid", n][:, -1]
164            size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
165            write = head_writes(model, tap, r["block"])[:, -1, r["head"]]
166            logits = ((write * model.norm.weight * size) @ model.lm_head.weight[ids].T).float()
167            rest = (logits * uninvolved).sum(1) / uninvolved.sum(1)
168            entry["steers"].append({"reader": name_of(r), "sender": name_of(s), "flipped": look(plain), "clean": look(clean_w),
169                                    "patched": look(attention.patterns(model, tap)),
170                                    "patched writes": {k: (logits[rows, who[k]] - rest).mean().item() for k in ("A", "B")}})
171
172    # sets: several heads moved together from the clean prompt into the flipped run
173    def together(flat):
174        edits = {}
175        for i in flat:
176            b, h = divmod(i, H)
177            k = ("heads", b + 1)
178            edit = patch.swap(store["clean"][k], head=h)
179            edits[k] = (lambda x, first=edits[k], second=edit: second(first(x))) if k in edits else edit
180        return patch.share(ld(patch.run(model, batch.flipped, edits)), base, goal)
181
182    mine = [(x["block"] - 1) * H + x["head"] for x in heads]
183    carriers = [(x["block"] - 1) * H + x["head"] for x in heads if x["carry"] >= 0.05]
184    entry["sets"] = {
185        "the heads": together(mine), "the heads that carry": together(carriers),
186        "all other heads": together([i for i in range(n * H) if i not in mine]), "all heads": together(list(range(n * H))),
187        "random": [together(s) for s in random_sets],
188    }
189    result["templates"].append(entry)
190    print(batch.template, {k: round(v, 2) for k, v in entry["sets"].items() if k != "random"})
191
192mean = lambda values: sum(values) / len(values)
193T = result["templates"]
194print("\nwhere each head reads from, at the position where it acts (clean | flipped | neutral)")
195for x in heads:
196    a = lambda v, k: mean([t["attention"][name_of(x)][v][k] for t in T])
197    print(f"  {name_of(x):15s} at {x['acts at']:3s}  " + " | ".join(" ".join(f"{k} {a(v, k):.2f}" for k in ("A", "B", "stop", "S")) for v in VERSIONS))
198print("\nwhat each head at the last position adds to a name's logit, against the nine names that are none of A, B and C (clean | flipped | neutral)")
199for x in readers:
200    wv = lambda v, k: mean([t["writes"][name_of(x)][v][k] for t in T])
201    print(f"  {name_of(x):15s}  " + " | ".join(" ".join(f"{k} {wv(v, k):+.2f}" for k in ("A", "B", "C")) for v in VERSIONS))
202print("\nwhere the writes of the heads at the last position come from, on clean and flipped prompts (to A, to B)")
203for x in readers:
204    sv = lambda v, k, m: mean([t["sources"][name_of(x)][v][k][m] for t in T])
205    print(f"  {name_of(x):15s}  " + " | ".join(" ".join(f"{k} {sv(v, k, 'A'):+.2f},{sv(v, k, 'B'):+.2f}" for k in ("A", "B", "stop", "S", "elsewhere")) for v in ("clean", "flipped")))
206print("\nholds: an earlier head's share alone, with a later head held, and relayed by the later head alone (only where held changes by 0.03 or more)")
207for i in range(len(T[0]["holds"])):
208    alone, held, relayed = (mean([t["holds"][i][k] for t in T]) for k in ("alone", "held", "relayed"))
209    if abs(alone - held) >= 0.03:
210        print(f"  {T[0]['holds'][i]['sender']:15s} {alone:5.2f} -> {held:5.2f} with {T[0]['holds'][i]['receiver']} held; relayed {relayed:5.2f}")
211print("\nsteers: where a head at the last position looks in the flipped run (A, B), with one earlier head from the clean run (only shifts of 0.05 or more)")
212for i in range(len(T[0]["steers"])):
213    g = lambda which, k: mean([t["steers"][i][which][k] for t in T])
214    if abs(g("patched", "A") - g("flipped", "A")) + abs(g("patched", "B") - g("flipped", "B")) >= 0.05:
215        s = T[0]["steers"][i]
216        print(f"  {s['reader']:15s} flipped A {g('flipped', 'A'):.2f} B {g('flipped', 'B'):.2f} | with {s['sender']} clean: A {g('patched', 'A'):.2f} B {g('patched', 'B'):.2f} | clean run A {g('clean', 'A'):.2f} B {g('clean', 'B'):.2f}")
217sets = {k: mean([t["sets"][k] for t in T]) for k in T[0]["sets"] if k != "random"}
218rand = torch.tensor([t["sets"]["random"] for t in T]).mean(0)
219print(f"\nsets of heads moved together: {', '.join(f'{k} {v:.2f}' for k, v in sets.items())}")
220print(f"  {len(random_sets)} random sets of {len(heads)} heads: median {rand.median():.2f}, 95th percentile {rand.quantile(0.95):.2f}, largest {rand.max():.2f}")
221
222for t in T:  # the record keeps means over the prompts; round them
223    t["sets"]["random"] = rounded(torch.tensor(t["sets"]["random"]), 3)
224save(f"runs/circuit/circuit-{meta['name']}.json", json.loads(json.dumps(result), parse_float=lambda s: round(float(s), 4)))