minterp

scripts/features.py

328 lines · View on GitHub · Linked from 6. Feature circuit

1"""
2Step 6: which latents does the answer depend on? See docs/06-feature-circuit.md.
3
4Uses the dictionary of step 5 (out/sae/<model>-b4.pt), the heads of step 4
5and the latents step 5 found for the names. Everything happens at the
6acting name's position after block 4, where step 3 found that the answer's
7dependence on who acts sits.
8
9parts       the stream there is what the dictionary rebuilds plus what it
10            misses. Each part is moved from one prompt into another, for
11            real: how much of who acts does each carry, and how much of a
12            name (step 3's two name experiments)?
13estimate    the same for every latent at once, estimated from one gradient
14check       the latents the estimate ranks highest, moved for real, one to
15            thirty-two of them, and the estimate set against the real
16            effect latent by latent
17head        what the veto head of step 4 adds to the acting name: against
18            the nine uninvolved names, as in step 4, and against the
19            uninvolved names of its own kind, where a kind is the six names
20            the model follows with "She" or the six it follows with "He";
21            what the gradient says this rests on, and what really happens
22            when latents are switched off
23names       for each name, two latents: the one step 5 picked by how often
24            it fires, and the one the veto head rests on most. What moving
25            just the two names' latents carries, by the kinds of the pair
26
27    uv run scripts/features.py
28    uv run scripts/features.py --model seed1
29    uv run scripts/features.py --dict s1                  # step 8: another dictionary
30    uv run scripts/features.py --model seed2 --head 6.5   # step 8: another head than the one with the largest share
31"""
32
33import argparse
34import json
35
36import torch
37
38from minterp import attribute, tasks
39from minterp import sae as S
40from minterp.common import load, rounded, save
41from minterp.tap import Tap, head_writes
42
43parser = argparse.ArgumentParser()
44parser.add_argument("--model", default="seed0", help="seed0, seed1 or seed2: the steps before must have run on it")
45parser.add_argument("--block", type=int, default=4)
46parser.add_argument("--dict", default="", help="another dictionary of the same place, by the name scripts/sae.py gave it (step 8)")
47parser.add_argument("--head", default="", help="another head read at the last position, as block.head, in place of the one with the largest share (step 8)")
48args = parser.parse_args()
49
50model, tok, meta = load(args.model)
51device = next(model.parameters()).device
52n = model.cfg.n_layers
53sae = S.load(S.path(meta["name"], args.block, args.dict), device)
54tag = f"-{args.dict}" if args.dict else ""  # the records of another dictionary are kept beside the first
55key = ("resid", args.block)
56with open(f"runs/sae/sae-{meta['name']}{tag}.json") as f:
57    step5 = json.load(f)["task"]
58name_latent = {row["name"]: row["latent"] for row in step5["names"]["acts"]}  # step 5's latent for each name as the one that acts
59with open(f"runs/circuit/circuit-{meta['name']}.json") as f:
60    veto = max((h for h in json.load(f)["heads"] if h["acts at"] == "end"), key=lambda h: h["carry"])
61if args.head:  # the record of another head is kept beside the first, as that of another dictionary is
62    veto = dict(zip(("block", "head"), map(int, args.head.split("."))))
63people = tasks.names(tok)
64name_ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
65she = torch.tensor([p in step5["names that go on with She"] for p in people], device=device)  # the kind of every name
66COUNTS = (1, 2, 4, 8, 16, 32)
67batches = tasks.batches(tok, device, people=people)
68result = {"model": meta, "block": args.block, "dictionary": args.dict or "first", "counts": COUNTS, "veto head": f"block {veto['block']} head {veto['head']}",
69          "names that go on with She": step5["names that go on with She"], "templates": []}
70
71
72def index(batch, i):
73    return torch.tensor([people.index(t[i]) for t in batch.names], device=device)
74
75
76def head_names(tap):
77    """What the veto head adds at the last position to the logit of each of the twelve names: (prompts, names), on the
78    scale of the final stream, whose size is held fixed so that the gradient is that of the head's write alone."""
79    final = tap["resid", n][:, -1].detach()
80    size = torch.rsqrt(final.pow(2).mean(-1, keepdim=True) + model.norm.eps)
81    write = head_writes(model, tap, veto["block"])[:, -1, veto["head"]]
82    return ((write * model.norm.weight * size) @ model.lm_head.weight[name_ids].T).float()
83
84
85def masks(batch):
86    """For every prompt: its row index, and the nine names that are none of A, B and C."""
87    rows = torch.arange(len(batch), device=device)
88    nine = torch.ones(len(batch), len(people), dtype=torch.bool, device=device)
89    for i in range(3):
90        nine[rows, index(batch, i)] = False
91    return rows, nine
92
93
94def against(names, target, pool):
95    """A name's logit minus the mean over a pool of names, per prompt."""
96    rows = torch.arange(names.shape[0], device=names.device)
97    return names[rows, target] - (names * pool).sum(1) / pool.sum(1)
98
99
100def adds_to(batch, target):
101    """A measure for attribute.through: what the veto head adds to one name per prompt, against the nine uninvolved names."""
102    _, nine = masks(batch)
103    return lambda logits, tap: against(head_names(tap), target, nine)
104
105
106# For each name: the latent the veto head rests on most when that name acts, over the clean and flipped prompts of all
107# patterns, and how many latents mark the name as the acting one by firing (on 90% of its prompts and 10% of the others).
108tops, actors, fired, where = [], [], [], []
109for p, batch in enumerate(batches):
110    for version, col in (("clean", 0), ("flipped", 1)):
111        actor = index(batch, col)
112        z, _, g, _, _ = attribute.through(model, sae, getattr(batch, version), args.block, adds_to(batch, actor), keep=("heads", "resid"))
113        at = batch.pos["S"]
114        tops.append((z[:, at] * g[:, at]).abs().argmax(1).cpu())
115        actors.append(actor.cpu())
116        fired.append((z[:, at] > 0).cpu())
117        where.append(torch.full((len(batch),), p))
118tops, actors, fired, where = torch.cat(tops), torch.cat(actors), torch.cat(fired), torch.cat(where)
119head_latent, result["names"] = {}, []
120for i, name in enumerate(people):
121    mine = actors == i
122    mode = tops[mine].mode().values.item()
123    marks = (fired[mine].float().mean(0) >= 0.9) & (fired[~mine].float().mean(0) <= 0.1)
124    head_latent[name] = mode
125    # Found on four patterns, does it hold on the fifth?
126    left_out = []
127    for p in range(len(batches)):
128        seen, unseen = mine & (where != p), mine & (where == p)
129        left_out.append((tops[unseen] == tops[seen].mode().values).float().mean().item())
130    result["names"].append({
131        "name": name, "step 5's latent": name_latent[name], "the head's latent": mode,
132        "the head rests on it most in": (tops[mine] == mode).float().mean().item(),
133        "on a pattern left out": sum(left_out) / len(left_out),
134        "latents that mark the name by firing": int(marks.sum()),
135        "the head's latent marks the name by firing": bool(marks[mode]), "step 5's latent marks the name by firing": bool(marks[name_latent[name]]),
136        "names for which the head's latent fires on 90% or more": [people[j] for j in range(len(people)) if fired[actors == j][:, mode].float().mean() >= 0.9],
137    })
138she_latents = {head_latent[p] for p in step5["names that go on with She"]}
139result["one latent for the six"] = sorted(she_latents)
140# One for all six in the first dictionaries of seed0 and seed1. Another dictionary or another model may give a name a
141# latent of its own: the shared one is then the one most of the six have, and it alone is switched off below.
142theirs = [head_latent[p] for p in step5["names that go on with She"]]
143the_shared = max(sorted(she_latents), key=theirs.count)
144result["the shared latent"] = the_shared
145
146for batch in batches:
147    s = batch.pos["S"]
148    rows, nine = masks(batch)
149    A, B, C = (index(batch, i) for i in range(3))
150    ld = lambda logits, tap=None: tasks.logit_diff(logits.float(), batch.right, batch.wrong)
151    zc, ec, _, _, vc = attribute.through(model, sae, batch.clean, args.block, ld)
152    zf, ef, gz, ge, vf = attribute.through(model, sae, batch.flipped, args.block, ld)  # gradients in the flipped run, the target
153    gap = (vc.mean() - vf.mean()).item()
154
155    @torch.no_grad()
156    def run(tokens, z, err, keep=()):
157        """A forward pass with the stream after the block set to decode(z) + err."""
158        with Tap(model, edit={key: attribute.rebuilt_with(sae, z, err)}, keep=keep) as tap:
159            logits, _ = model(tokens)
160        return logits, tap
161
162    moved = lambda z, err: ld(run(batch.flipped, z, err)[0])
163    share = lambda values, pick=slice(None): ((values[pick].mean() - vf[pick].mean()) / (vc[pick].mean() - vf[pick].mean())).item()
164
165    def at_s(target, source):
166        out = target.clone()
167        out[:, s] = source[:, s]
168        return out
169
170    entry = {"template": batch.template, "gap": gap}
171    # parts: what the dictionary rebuilds, what it misses, and both, moved for real
172    entry["parts"] = {"rebuilt": share(moved(at_s(zf, zc), ef)), "missed": share(moved(zf, at_s(ef, ec))), "both": share(moved(at_s(zf, zc), at_s(ef, ec)))}
173    # The same split for a name, as in step 3: the prompt with C in place of a name, moved into the prompt it was made from.
174    entry["name parts"] = {}
175    for label, source, target, old in (("second name", batch.other, batch.clean, B), ("first name", batch.other_first, batch.flipped, A)):
176        to_c = lambda logits, old=old: tasks.logit_diff(logits.float(), name_ids[C], name_ids[old])  # token ids, not places in the list
177        zo, eo, _, _, _ = attribute.through(model, sae, source, args.block, lambda logits, tap: to_c(logits))
178        zt, et, _, _, _ = attribute.through(model, sae, target, args.block, lambda logits, tap: to_c(logits))
179        base, goal = to_c(run(target, zt, et)[0]).mean(), to_c(run(source, zo, eo)[0]).mean()
180        way = lambda z, err: ((to_c(run(target, z, err)[0]).mean() - base) / (goal - base)).item()
181        entry["name parts"][label] = {"rebuilt": way(at_s(zt, zo), et), "missed": way(zt, at_s(et, eo)), "both": way(at_s(zt, zo), at_s(et, eo))}
182
183    # estimate: every latent at the acting name at once
184    est = (zc[:, s] - zf[:, s]) * gz[:, s]  # (prompts, width): the estimated change of the logit difference, in logits
185    est_missed = ((ec[:, s] - ef[:, s]) * ge[:, s]).sum(1)
186    on_c, on_f = zc[:, s] > 0, zf[:, s] > 0
187    order = est.abs().argsort(dim=1, descending=True)
188    entry["estimate"] = {"rebuilt": (est.sum(1).mean() / gap).item(), "missed": (est_missed.mean() / gap).item(),
189                         "latents that differ": (zc[:, s] != zf[:, s]).sum(1).float().mean().item(),
190                         "on in the clean prompt only": (on_c & ~on_f).sum(1).float().mean().item(),
191                         "on in the flipped prompt only": (~on_c & on_f).sum(1).float().mean().item(),
192                         "on in both": (on_c & on_f & (zc[:, s] != zf[:, s])).sum(1).float().mean().item()}
193
194    # check: the highest-ranked latents moved for real
195    by_count = {"estimated": [], "real": []}
196    for count in COUNTS:
197        top = order[:, :count]
198        z = zf.clone()
199        z[rows[:, None], s, top] = zc[rows[:, None], s, top]
200        by_count["real"].append(share(moved(z, ef)))
201        by_count["estimated"].append((est.gather(1, top).sum(1).mean() / gap).item())
202    entry["by count"] = by_count
203    # latent by latent: the estimate against the real effect, for each prompt's five highest
204    pairs = []
205    for rank in range(5):
206        j = order[:, rank]
207        z = zf.clone()
208        z[rows, s, j] = zc[rows, s, j]
209        pairs.append(torch.stack([est[rows, j], moved(z, ef) - vf]))  # (2, prompts): estimated and real change, in logits
210    pairs = torch.stack(pairs)  # (ranks, 2, prompts)
211    flat = pairs.permute(1, 0, 2).reshape(2, -1)
212    entry["one latent"] = {"estimated": rounded(pairs[:, 0].mean(1), 3), "real": rounded(pairs[:, 1].mean(1), 3),
213                           "correlation": torch.corrcoef(flat)[0, 1].item(), "same sign": (flat[0].sign() == flat[1].sign()).float().mean().item()}
214
215    # names: the two names' latents, by step 5's choice and by the head's, overall and by the kinds of the pair
216    kinds = {"both She": she[A] & she[B], "both He": ~she[A] & ~she[B], "one of each": she[A] != she[B]}
217    entry["pairs"] = {k: {"prompts": int(m.sum()), "gap": (vc[m].mean() - vf[m].mean()).item()} for k, m in kinds.items()}
218    entry["names"] = {}
219    for choice, table in (("step 5's latents", name_latent), ("the head's latents", head_latent)):
220        la = torch.tensor([table[a] for a, _, _ in batch.names], device=device)  # the first name acts on the clean prompt
221        lb = torch.tensor([table[b] for _, b, _ in batch.names], device=device)  # the second on the flipped one
222        z = zf.clone()
223        z[rows, s, la] = zc[rows, s, la]
224        z[rows, s, lb] = zc[rows, s, lb]
225        values = moved(z, ef)
226        hit = (order[:, 0] == la) | (order[:, 0] == lb)
227        entry["names"][choice] = {"moved for real": share(values), "estimated": ((est[rows, la] + est[rows, lb]).mean() / gap).item(),
228                                  "the highest is one of them": hit.float().mean().item(),
229                                  "by pair": {k: {"moved for real": share(values, m), "the highest is one of them": hit[m].float().mean().item()} for k, m in kinds.items()}}
230
231    # head: what the veto head adds to the acting name on clean prompts, against two pools of uninvolved names
232    own = nine & (she[None, :] == she[A][:, None])  # the uninvolved names of the acting name's kind
233    other_kind = nine & ~own
234    ha = torch.tensor([head_latent[a] for a, _, _ in batch.names], device=device)
235    z, err, g, gerr, value = attribute.through(model, sae, batch.clean, args.block, adds_to(batch, A), keep=("heads", "resid"))
236    rest_on = z[:, s] * g[:, s]  # what the number would lose, to first order, if a latent were off
237    ranked = rest_on.abs().argsort(dim=1, descending=True)
238    plain = head_names(run(batch.clean, z, err, keep=("heads", "resid"))[1])
239
240    def off(mask):
241        """The head's write with some latents at the acting name switched off, for real."""
242        cut = z.clone()
243        cut[:, s] = cut[:, s] * (~mask)
244        return head_names(run(batch.clean, cut, err, keep=("heads", "resid"))[1])
245
246    one_hot = lambda j: torch.zeros_like(z[:, s], dtype=torch.bool).scatter_(1, j[:, None], True)
247    everything = torch.ones_like(z[:, s], dtype=torch.bool)
248    shared = torch.zeros_like(z[:, s], dtype=torch.bool)
249    shared[:, the_shared] = True
250    acts_she = she[A]
251    kind_gap = lambda names: (names * own).sum(1) / own.sum(1) - (names * other_kind).sum(1) / other_kind.sum(1)
252    head = {"by kind of the acting name": {}}
253    for label, m in (("She", acts_she), ("He", ~acts_she)):
254        head["by kind of the acting name"][label] = {
255            "prompts": int(m.sum()),
256            "against the nine": against(plain, A, nine)[m].mean().item(),
257            "against its own kind": against(plain, A, own)[m].mean().item(),
258            "its own kind against the other": kind_gap(plain)[m].mean().item(),
259        }
260    cut = off(shared)
261    m = acts_she
262    head["the shared latent switched off, where a She name acts"] = {
263        "the acting name against the nine": (against(cut, A, nine) - against(plain, A, nine))[m].mean().item(),
264        "estimated": -rest_on[:, the_shared][m].mean().item(),
265        "the acting name against its own kind": (against(cut, A, own) - against(plain, A, own))[m].mean().item(),
266        "its own kind against the other": (kind_gap(cut) - kind_gap(plain))[m].mean().item(),
267    }
268    head["adds to the acting name"] = value.mean().item()
269    head["estimated to rest on"] = {
270        "the latents at the acting name": rest_on.sum(1).mean().item(),
271        "what the dictionary misses there": (err[:, s] * gerr[:, s]).sum(1).mean().item(),
272        "the other positions": ((z * g).sum((1, 2)) + (err * gerr).sum((1, 2)) - rest_on.sum(1) - (err[:, s] * gerr[:, s]).sum(1)).mean().item(),
273        "its highest latent": rest_on.gather(1, ranked[:, :1]).mean().item(),
274        "the latents below zero": rest_on.clamp(max=0).sum(1).mean().item(), "the latents above zero": rest_on.clamp(min=0).sum(1).mean().item(),
275    }
276    head["really lost when switched off"] = {
277        "its highest latent": -(against(off(one_hot(ranked[:, 0])), A, nine) - value).mean().item(),
278        "all latents at the acting name": -(against(off(everything), A, nine) - value).mean().item(),
279    }
280    head["its highest latent is the head's latent for the name"] = (ranked[:, 0] == ha).float().mean().item()
281    # The other name: what the head adds, by whether the two names are of one kind.
282    z2, err2, g2, _, value2 = attribute.through(model, sae, batch.clean, args.block, adds_to(batch, B), keep=("heads", "resid"))
283    rest2 = z2[:, s] * g2[:, s]
284    top2 = rest2.gather(1, rest2.abs().argsort(dim=1, descending=True)[:, :1])[:, 0]
285    same = she[A] == she[B]
286    cut_all = against(off(everything), B, nine)
287    head["the other name"] = {
288        "adds": value2.mean().item(), "adds, same kind": value2[same].mean().item(), "adds, other kind": value2[~same].mean().item(),
289        "estimated to rest on the latents at the acting name": rest2.sum(1).mean().item(),
290        "really lost when all latents there are switched off": -(cut_all - value2).mean().item(),
291        "its highest latent, size": top2.abs().mean().item(), "its highest latent, same kind": top2[same].mean().item(), "its highest latent, other kind": top2[~same].mean().item(),
292    }
293    entry["head"] = head
294    result["templates"].append(entry)
295    print(batch.template, {k: round(v, 2) for k, v in entry["parts"].items()})
296
297T = result["templates"]
298mean = lambda pick: sum(pick(t) for t in T) / len(T)
299show = lambda d, digits=2: ", ".join(f"{k} {v:+.{digits}f}" for k, v in d.items() if isinstance(v, float))
300print(f"\nat the acting name after block {args.block} (shares of a gap, moved for real):")
301print(f"  who acts: what the dictionary rebuilds {mean(lambda t: t['parts']['rebuilt']):.2f}, what it misses {mean(lambda t: t['parts']['missed']):.2f}, both {mean(lambda t: t['parts']['both']):.2f}; "
302      f"estimated {mean(lambda t: t['estimate']['rebuilt']):.2f} and {mean(lambda t: t['estimate']['missed']):.2f}")
303for label in ("second name", "first name"):
304    print(f"  the {label}: rebuilt {mean(lambda t: t['name parts'][label]['rebuilt']):.3f}, missed {mean(lambda t: t['name parts'][label]['missed']):.3f}, both {mean(lambda t: t['name parts'][label]['both']):.3f}")
305e = {k: mean(lambda t, k=k: t["estimate"][k]) for k in T[0]["estimate"]}
306print(f"  {e['latents that differ']:.0f} latents differ: {e['on in the clean prompt only']:.0f} on in the clean prompt only, {e['on in the flipped prompt only']:.0f} in the flipped only, {e['on in both']:.0f} in both")
307print("  the highest-ranked latents:  " + " ".join(f"{c:>6}" for c in COUNTS))
308print("    estimated                  " + " ".join(f"{mean(lambda t, i=i: t['by count']['estimated'][i]):6.2f}" for i in range(len(COUNTS))))
309print("    moved for real             " + " ".join(f"{mean(lambda t, i=i: t['by count']['real'][i]):6.2f}" for i in range(len(COUNTS))))
310print(f"  one latent at a time: correlation {mean(lambda t: t['one latent']['correlation']):.2f}, same sign {100 * mean(lambda t: t['one latent']['same sign']):.1f}%")
311for choice in T[0]["names"]:
312    nm = T[0]["names"][choice]
313    print(f"  {choice}: moved for real {mean(lambda t: t['names'][choice]['moved for real']):.2f}, the highest is one of them {100 * mean(lambda t: t['names'][choice]['the highest is one of them']):.0f}%; by pair: "
314          + "; ".join(f"{k} {mean(lambda t, k=k: t['names'][choice]['by pair'][k]['moved for real']):+.2f} ({100 * mean(lambda t, k=k: t['names'][choice]['by pair'][k]['the highest is one of them']):.0f}%)" for k in nm["by pair"]))
315print("  gap by pair: " + ", ".join(f"{k} {mean(lambda t, k=k: t['pairs'][k]['gap']):.2f} ({T[0]['pairs'][k]['prompts']} prompts)" for k in T[0]["pairs"]))
316print("  per name: step 5's latent / the head's (share of prompts; on a pattern left out), names the head's latent fires for")
317for x in result["names"]:
318    print(f"    {x['name']:5s} {x['step 5' + chr(39) + 's latent']:4d} / {x['the head' + chr(39) + 's latent']:4d} ({100 * x['the head rests on it most in']:.0f}%; {100 * x['on a pattern left out']:.0f}%) {', '.join(x['names for which the head' + chr(39) + 's latent fires on 90% or more'])}")
319for label in ("She", "He"):
320    print(f"  where a {label} name acts, {result['veto head']} adds to it: " + show({k: mean(lambda t, k=k: t['head']['by kind of the acting name'][label][k]) for k in T[0]['head']['by kind of the acting name'][label] if k != 'prompts'}))
321k1 = "the shared latent switched off, where a She name acts"
322print(f"  {k1}: " + show({k: mean(lambda t, k=k: t['head'][k1][k]) for k in T[0]['head'][k1]}))
323print(f"  adds to the acting name {mean(lambda t: t['head']['adds to the acting name']):+.2f}; estimated to rest on: " + show({k: mean(lambda t, k=k: t['head']['estimated to rest on'][k]) for k in T[0]['head']['estimated to rest on']}))
324print("  really lost when switched off: " + show({k: mean(lambda t, k=k: t['head']['really lost when switched off'][k]) for k in T[0]['head']['really lost when switched off']})
325      + f"; its highest latent is the head's latent for the name in {100 * mean(lambda t: t['head']['its highest latent is the head' + chr(39) + 's latent for the name']):.0f}%")
326print("  the other name: " + show({k: mean(lambda t, k=k: t['head']['the other name'][k]) for k in T[0]['head']['the other name']}))
327
328save(f"runs/features/features-{meta['name']}{tag}{'-h' + args.head if args.head else ''}.json", result)