minterp

scripts/open.py

187 lines · View on GitHub · Linked from 8. Limits

1"""
2Step 8: the same task in a published model. See docs/08-limits.md.
3
4Qwen2.5-0.5B-Instruct, read from its published weights by minllm's stage 5
5loader: 24 blocks, 14 heads, width 896, 0.5 billion parameters. It goes
6into the same Transformer class as the story model, so the scripts of
7steps 1 and 3 run on it as they are:
8
9    uv run scripts/behavior.py --model qwen
10    uv run scripts/lens.py --model qwen
11    uv run scripts/patch.py --model qwen --stream-only   # minutes; without the flag every part is moved, in hours
12    uv run scripts/heads.py                              # what the heads found by the full run read and write
13
14This script repeats the first experiment of step 7 there, without a
15dictionary, at the acting name's position.
16
17kinds    which names the model follows with " She" and which with " He",
18         found as step 5 found them
19steer    on a clean prompt, after the block in the middle of the model
20         (--block): the directions of step 7 that need no dictionary and
21         its controls, at doses from -2 to 2. The random control is as
22         long as the pair direction.
23depth    at dose 1 after every second block: the pair direction, beside
24         what step 3's patch at the acting name carries at that depth, and
25         the names' means, the vector every flipped prompt shares, and the
26         two together
27
28    uv run scripts/open.py
29    uv run scripts/open.py --model seed0 --block 4   # the story models, for the comparison by depth
30    uv run scripts/open.py --model seed1 --block 4
31"""
32
33import argparse
34import json
35
36import torch
37
38from minterp import steer, tasks
39from minterp.common import eot, load, rounded, save
40from minterp.tap import Tap
41
42parser = argparse.ArgumentParser()
43parser.add_argument("--model", default="qwen")
44parser.add_argument("--block", type=int, default=None, help="where to steer: by default after the block in the middle")
45args = parser.parse_args()
46
47model, tok, meta = load(args.model)
48device = next(model.parameters()).device
49n = model.cfg.n_layers
50block = args.block if args.block is not None else n // 2
51people = tasks.names(tok)
52k = len(people)
53name_ids = torch.tensor([tok.encode_ordinary(" " + p)[0] for p in people], device=device)
54DOSES = (-2.0, -1.0, -0.5, 0.0, 0.25, 0.5, 0.75, 1.0, 1.5, 2.0)
55generator = torch.Generator().manual_seed(0)
56more = torch.Generator().manual_seed(1)  # for a control added later, so that the first keeps its draws
57series = lambda values: rounded(torch.tensor(values))
58
59# kinds: after a sentence about one name, does the model go on with " She" or " He"?
60she_id, he_id = (tok.encode_ordinary(w)[0] for w in (" She", " He"))
61with torch.no_grad():
62    after = model(torch.tensor([[eot(tok)] + tok.encode_ordinary(f"One day, {p} went to the park.") for p in people], device=device))[0][:, -1].float()
63pronoun = after[:, she_id] - after[:, he_id]
64she = pronoun > 0
65result = {"model": meta, "block": block, "doses": DOSES, "names": people, "names that go on with She": [p for p, g in zip(people, she.tolist()) if g],
66          "logit of She minus logit of He after each name": rounded(pronoun, 2)}
67
68batches = tasks.batches(tok, device, people=people)
69A, B, C = (torch.tensor([people.index(t[i]) for t in batches[0].names], device=device) for i in range(3))
70rows = torch.arange(len(A), device=device)
71acts = torch.stack([A, B])
72D, E = steer.stand_ins(A, B, C, she)
73apart = torch.ones(len(A), k, dtype=torch.bool, device=device)
74for names in (A, B, C, D, E):
75    apart[rows, names] = False
76own_kind = {"A": apart & (she[None] == she[A][:, None]), "B": apart & (she[None] == she[B][:, None])}
77additive = steer.term_per_name(A, B, k)
78kinds = {"both She": she[A] & she[B], "both He": ~she[A] & ~she[B], "one of each": she[A] != she[B]}
79result["pairs"] = {g: int(m.sum()) for g, m in kinds.items()}
80
81# The stream at the acting name after every block, and the plain answers, for the clean and the flipped prompt.
82at_name, plain, third = [], [], []
83for batch in batches:
84    per = []
85    for version in ("clean", "flipped"):
86        with torch.no_grad(), Tap(model, keep=("resid",)) as tap:
87            logits, _ = model(getattr(batch, version))
88        per.append((torch.stack([tap["resid", d][:, batch.pos["S"]].float() for d in range(n + 1)]), logits[:, -1].float()))
89    at_name.append(torch.stack([x for x, _ in per]))
90    plain.append(torch.stack([l for _, l in per]))
91    with torch.no_grad():
92        third.append(model(batch.neutral)[0][:, -1].float())
93at_name, plain = torch.stack(at_name), torch.stack(plain)  # (patterns, versions, depths, prompts, dim) and (patterns, versions, prompts, vocab)
94
95
96def directions_at(p, depth, controls=True):
97    """The directions of step 7 for pattern p after one block, built on the other four patterns."""
98    others = [q for q in range(len(batches)) if q != p]
99    x = at_name[others][:, :, depth]  # (patterns, versions, prompts, dim)
100    actor = acts[None].expand(len(others), -1, -1)
101    mean_when = torch.stack([x[actor == i].mean(0) for i in range(k)])
102    pair = (x[:, 1] - x[:, 0]).mean(0)
103    order = pair.mean(0, keepdim=True).expand_as(pair)
104    means = mean_when[B] - mean_when[A]
105    if not controls:
106        return {"pair": pair, "means": means, "order": order, "means + order": means + order}
107    here = at_name[p][:, depth]
108    return {"own": here[1] - here[0], "pair": pair, "names": additive @ pair, "means + order": means + order, "means": means, "order": order,
109            "kind": steer.kind_only(mean_when, A, B, she), "other names": mean_when[D] - mean_when[E],
110            "random, long": steer.like(pair, generator), "means + random": means + steer.like(order, more), "wipe": here.mean((0, 1))[None] - here[0]}
111
112
113OUTSIDE = "to the flipped prompt, outside the two names"
114NAMES = ("own", "pair", "names", "means + order", "means", "order", "kind", "other names", "random, long", "means + random", "wipe")
115BY_DEPTH = ("pair", "means", "order", "means + order")
116result["directions"], result["templates"] = NAMES, []
117for p, batch in enumerate(batches):
118    ld = lambda logits: tasks.logit_diff(logits, batch.right, batch.wrong)
119    clean, flipped = ld(plain[p, 0]), ld(plain[p, 1])
120    gap = (clean - flipped).mean()
121    names_out = torch.stack([batch.right, batch.wrong], dim=1)
122    own = {x: lambda logits, x=x, t=t: steer.against(logits[:, name_ids], t, own_kind[x]) for x, t in (("A", A), ("B", B))}
123    between = steer.elsewhere(plain[p, 1], plain[p, 0], names_out)
124    directions = directions_at(p, block)
125    entry = {"template": batch.template, "gap": gap.item(), "stream size at the acting name": at_name[p, 0, block].norm(dim=1).mean().item(),
126             "pairs": {g: {"gap": (clean - flipped)[m].mean().item()} for g, m in kinds.items()},
127             "length": {name: {"all": d.norm(dim=1).mean().item(), **{g: d[m].norm(dim=1).mean().item() for g, m in kinds.items()}} for name, d in directions.items()},
128             "plain": {"clean": clean.mean().item(), "flipped": flipped.mean().item(), "a third name acts, share": ((clean - ld(third[p])).mean() / gap).item(),
129                       "the flipped prompt turned": (flipped < 0).float().mean().item(), "the clean prompt turned": (clean < 0).float().mean().item(),
130                       "from the clean prompt to the flipped, outside the two names": between.mean().item(),
131                       "the same, by pair": {g: between[m].mean().item() for g, m in kinds.items()}},
132             "steered": {}, "depth": {name: {"share": [], "turned": []} for name in BY_DEPTH}}
133    for name, d in directions.items():
134        out = {m: [] for m in ("share", "turned", "A against its own kind", "B against its own kind", OUTSIDE)}
135        by = {g: [] for g in kinds}
136        for dose in DOSES:
137            logits = steer.steered(model, batch.clean, block, d, [batch.pos["S"]], dose)
138            moved = clean - ld(logits)
139            out["share"].append((moved.mean() / gap).item())
140            out["turned"].append((ld(logits) < 0).float().mean().item())
141            for x in ("A", "B"):
142                out[f"{x} against its own kind"].append(((own[x](logits) - own[x](plain[p, 0])).mean() / (own[x](plain[p, 1]) - own[x](plain[p, 0])).mean()).item())
143            out[OUTSIDE].append(steer.elsewhere(plain[p, 1], logits, names_out).mean().item())
144            for g, m in kinds.items():
145                by[g].append(moved[m].mean().item())
146        entry["steered"][name] = {**{m: series(v) for m, v in out.items()}, "moved, in logits, by pair": {g: series(v) for g, v in by.items()}}
147    # depth: the pair direction at dose 1 after every second block
148    for depth in range(0, n + 1, 2):
149        for name, d in directions_at(p, depth, controls=False).items():
150            logits = steer.steered(model, batch.clean, depth, d, [batch.pos["S"]], 1.0)
151            entry["depth"][name]["share"].append(((clean - ld(logits)).mean() / gap).item())
152            entry["depth"][name]["turned"].append((ld(logits) < 0).float().mean().item())
153    result["templates"].append(entry)
154    print(batch.template, f"clean {clean.mean():+.2f}, flipped {flipped.mean():+.2f}")
155result["depths"] = list(range(0, n + 1, 2))
156try:  # what step 3's patch at the acting name carries at the same depths, if that script has run on this model
157    with open(f"runs/patch/patch-{meta['name']}.json") as f:
158        travel = json.load(f)
159    at = travel["travel groups"].index("S")
160    result["the patch at the acting name, by depth"] = rounded(torch.tensor([[t["travel"]["break"][d][at] for d in result["depths"]] for t in travel["templates"]]).mean(0), 3)
161except FileNotFoundError:
162    pass
163
164T = result["templates"]
165mean = lambda pick: sum(pick(t) for t in T) / len(T)
166row = lambda values: " ".join(f"{v:6.2f}" for v in values)
167at = DOSES.index(1.0)
168print(f"\n{meta['name']}: {n} blocks; goes on with She after {', '.join(result['names that go on with She'])}; steering at the acting name after block {block}, where the stream's size is {mean(lambda t: t['stream size at the acting name']):.1f}")
169print(f"plain: clean {mean(lambda t: t['plain']['clean']):+.2f}, flipped {mean(lambda t: t['plain']['flipped']):+.2f}; a third name acts {mean(lambda t: t['plain']['a third name acts, share']):.2f} of the way; A above B in "
170      f"{100 * mean(lambda t: t['plain']['the flipped prompt turned']):.1f}% of flipped and {100 * mean(lambda t: t['plain']['the clean prompt turned']):.1f}% of clean prompts; outside the two names the clean prompt is "
171      f"{mean(lambda t: t['plain']['from the clean prompt to the flipped, outside the two names']):.2f} nats from the flipped (" + ", ".join(f"{g} {mean(lambda t, g=g: t['plain']['the same, by pair'][g]):.2f}" for g in kinds) + ")")
172for measure in ("share", "turned"):
173    print(f"  {measure}\n    dose           " + row(DOSES))
174    for name in NAMES:
175        print(f"    {name:14s} " + row([mean(lambda t, i=i: t["steered"][name][measure][i]) for i in range(len(DOSES))]))
176print("  at dose 1: A and B against their own kind; nats from the flipped prompt; length; share of the gap by pair (" + ", ".join(f"{g} {mean(lambda t, g=g: t['pairs'][g]['gap']):.2f}" for g in kinds) + ")")
177for name in NAMES:
178    print(f"    {name:14s} {mean(lambda t: t['steered'][name]['A against its own kind'][at]):+.2f} {mean(lambda t: t['steered'][name]['B against its own kind'][at]):+.2f}; {mean(lambda t: t['steered'][name][OUTSIDE][at]):.2f}; "
179          f"{mean(lambda t: t['length'][name]['all']):.1f}; " + ", ".join(f"{mean(lambda t, g=g: t['steered'][name]['moved, in logits, by pair'][g][at]) / mean(lambda t, g=g: t['pairs'][g]['gap']):+.2f}" for g in kinds))
180print("  at dose 1 by depth, share of the way   " + " ".join(f"{d:>5}" for d in result["depths"]))
181for name in BY_DEPTH:
182    print(f"    {name:34s} " + " ".join(f"{mean(lambda t, i=i: t['depth'][name]['share'][i]):5.2f}" for i in range(len(result["depths"]))))
183print("    pair, A above B                    " + " ".join(f"{mean(lambda t, i=i: t['depth']['pair']['turned'][i]):5.2f}" for i in range(len(result["depths"]))))
184if "the patch at the acting name, by depth" in result:
185    print("    the patch of step 3                " + " ".join(f"{v:5.2f}" for v in result["the patch at the acting name, by depth"]))
186
187save(f"runs/open/steer-{meta['name']}.json", result)