minterp

scripts/probe.py

190 lines · View on GitHub · Linked from 2. Probe

1"""
2Step 2: where can the answer be read out? See docs/02-probe.md.
3
4For every depth (after the embedding and after every block) and five
5positions named by their role (A, B, S, the token after S, and the last
6one), this fits linear readers from the stream to four properties of the
7prompt, each a name out of twelve:
8
9    due      the name that is due          acts     the name that acts
10    first    the first name, A             second   the second name, B
11
12and to a fifth that is made up, `control`: a random name drawn once for
13every ordered pair, the same in both versions and all patterns. Nothing in
14the text determines it, so a reader can only get it right by remembering
15pairs it was trained on.
16
17Readers are trained on four sentence patterns and tested on the fifth, each
18pattern in turn, under two rules:
19
20    new    tested on name pairs that were held out of training as well
21    seen   tested on the same pairs it was trained on
22
23It also records, on the same test prompts, how often the lens of step 1
24ranks the due name first among the twelve; how well the due name is read by
25combining the readers for first, second and acts without training on it;
26and how well it can be read from the three tokens' identities alone, with
27no model at all.
28
29    uv run scripts/probe.py
30    uv run scripts/probe.py --model seed1
31"""
32
33import argparse
34import os
35
36import torch
37
38from minterp import lens, probe, tasks
39from minterp.common import load, rounded, save
40from minterp.tap import Tap
41
42parser = argparse.ArgumentParser()
43parser.add_argument("--model", default="seed0", help="seed0, seed1, or the path of a checkpoint")
44args = parser.parse_args()
45
46ROLES = ("A", "B", "S", "S+1", "end")
47TARGETS = ("due", "acts", "first", "second", "control")
48VERSIONS = ("clean", "flipped")
49
50model, tok, meta = load(args.model, device="cpu")  # float64 on the CPU: the same numbers on every machine
51people = tasks.names(tok)
52k, depth = len(people), model.cfg.n_layers + 1
53name_ids = torch.tensor([tok.encode_ordinary(" " + n)[0] for n in people])
54batches = tasks.batches(tok, people=people)
55
56# The stream at the five positions, and the lens's pick among the twelve names at the last one.
57streams, picks = [], []
58for batch in batches:
59    at = [batch.pos["A"], batch.pos["B"], batch.pos["S"], batch.pos["S"] + 1, batch.pos["end"]]
60    per_version, per_pick = [], []
61    for version in VERSIONS:
62        with Tap(model, keep=("resid",)) as tap, torch.no_grad():
63            model(getattr(batch, version))
64        stream = torch.stack([tap["resid", i][:, at] for i in range(depth)], dim=1)  # (prompts, depth, roles, dim)
65        per_version.append(stream)
66        per_pick.append(lens.decode(model, stream[:, :, -1])[..., name_ids].argmax(-1))  # (prompts, depth)
67    streams.append(torch.stack(per_version))
68    picks.append(torch.stack(per_pick))
69X = torch.stack(streams).double()  # (patterns, versions, prompts, depth, roles, dim)
70picks = torch.stack(picks)  # (patterns, versions, prompts, depth)
71patterns, prompts = X.shape[0], X.shape[2]
72
73# Labels, as indices into `people`. They are the same in every pattern.
74index = {n: i for i, n in enumerate(people)}
75first = torch.tensor([index[a] for a, _, _ in batches[0].names])
76second = torch.tensor([index[b] for _, b, _ in batches[0].names])
77control = torch.randint(0, k, (prompts,), generator=torch.Generator().manual_seed(0)).repeat(2, 1)
78labels = torch.stack([
79    torch.stack([second, first]),  # due: B on the clean prompt, A on the flipped one
80    torch.stack([first, second]),  # acts
81    torch.stack([first, first]),
82    torch.stack([second, second]),
83    control,
84])  # (targets, versions, prompts)
85
86# A third of the pairs, in both orders, is kept out of training under the rule "new".
87held = (first + second) % 3 == 0
88assert set(first[~held].tolist()) == set(second[~held].tolist()) == set(range(k)), "a name never trains"
89
90
91def rows(which, mask):
92    """The chosen patterns and pairs as rows: streams (places, rows, dim), labels, pattern and version of every row."""
93    x = X[which][:, :, mask]  # (patterns, versions, pairs, depth, roles, dim)
94    p, v, m = x.shape[:3]
95    x = x.permute(3, 4, 0, 1, 2, 5).reshape(depth * len(ROLES), p * v * m, -1)
96    lab = labels[:, :, mask][:, None].expand(-1, p, -1, -1).reshape(len(TARGETS), -1)
97    group = torch.tensor(which)[:, None, None].expand(p, v, m).reshape(-1)
98    version = torch.arange(v)[None, :, None].expand(p, v, m).reshape(-1)
99    return x, lab, group, version
100
101
102def by_version(right, version):
103    """(places, rows, ...) of hits -> (versions, depth, roles, ...) of accuracies."""
104    out = torch.stack([right[:, version == v].double().mean(1) for v in range(len(VERSIONS))])
105    return out.unflatten(1, (depth, len(ROLES)))
106
107
108result = {
109    "model": meta, "names": people, "roles": ROLES, "targets": TARGETS, "versions": VERSIONS, "penalties": probe.LAMS,
110    "pairs": {"held out": int(held.sum()), "trained": int((~held).sum())},
111}
112every = torch.ones(prompts, dtype=torch.bool)
113for rule, train_mask, test_mask in (("new", ~held, held), ("seen", every, every)):
114    acc, composite, lens_acc, chosen = [], [], [], []
115    for fold in range(patterns):
116        others = [p for p in range(patterns) if p != fold]
117        xtr, ltr, gtr, _ = rows(others, train_mask)
118        xte, lte, _, vte = rows([fold], test_mask)
119        out, pick = probe.read(xtr, ltr, gtr, xte, k)  # (places, rows, targets, k)
120        acc.append(by_version(out.argmax(-1) == lte.T, vte))  # (versions, depth, roles, targets)
121        chosen.append(pick.unflatten(0, (depth, len(ROLES))))
122        # The due name without a reader of its own: it is one of the two names, and not the one that acts.
123        t = TARGETS.index
124        combined = out[:, :, t("first")] + out[:, :, t("second")] - out[:, :, t("acts")]
125        composite.append(by_version(combined.argmax(-1) == lte[t("due")], vte))  # (versions, depth, roles)
126        due = labels[t("due")][:, test_mask]  # (versions, pairs)
127        lens_acc.append((picks[fold][:, test_mask] == due[:, :, None]).double().mean(1))  # (versions, depth)
128    result[rule] = {
129        "train rows": xtr.shape[1], "test rows": xte.shape[1],
130        "accuracy": rounded(torch.stack(acc)),  # (folds, versions, depth, roles, targets)
131        "composite": rounded(torch.stack(composite)),  # (folds, versions, depth, roles)
132        "lens": rounded(torch.stack(lens_acc)),  # (folds, versions, depth)
133        "penalty": torch.stack(chosen)[..., 0].tolist(),  # for the due name: (folds, depth, roles), an index into penalties
134    }
135
136# No model at all: can a linear reader produce the due name from the identities of the tokens?
137eye = torch.eye(k, dtype=torch.float64)
138tokens = {
139    "first, second and acts": lambda v: torch.cat([eye[first], eye[second], eye[labels[1, v]]], dim=1),
140    "first and second": lambda v: torch.cat([eye[first], eye[second]], dim=1),
141}
142result["tokens only"] = {}
143for name, make in tokens.items():
144    feats = torch.stack([make(v) for v in range(2)])  # (versions, prompts, dim)
145    xtr, xte = feats[:, ~held].reshape(1, -1, feats.shape[-1]), feats[:, held].reshape(1, -1, feats.shape[-1])
146    ltr, lte = labels[:1, :, ~held].reshape(1, -1), labels[0][:, held]
147    groups = torch.arange(xtr.shape[1]) % 4
148    out = probe.read(xtr, ltr, groups, xte, k)[0][0, :, 0]  # (test rows, k)
149    right = (out.argmax(-1).view(2, -1) == lte).double()
150    # Without the acting name the two versions of a prompt are the same input,
151    # so the reader gives the same answer to both and is right on one. Which
152    # one is a tie between two equal scores, so only the two together count.
153    top2 = out.topk(2, dim=-1).values
154    result["tokens only"][name] = {
155        "clean": right[0].mean().item(), "flipped": right[1].mean().item(), "both": right.mean().item(),
156        "tied": bool(((top2[:, 0] - top2[:, 1]).abs() < 1e-9).all()),
157    }
158
159path = save(f"runs/probe/probe-{meta['name']}.json", result)
160
161# The reader for the due name at the last position, fitted on every prompt, for later steps. Not part of the record.
162x, lab, _, _ = rows(list(range(patterns)), every)
163end = [d * len(ROLES) + ROLES.index("end") for d in range(depth)]
164mean, spread = x[end].mean(1, keepdim=True), x[end].std(1, keepdim=True).clamp_min(1e-8)
165W, b = probe.fit((x[end] - mean) / spread, probe.onehot(lab[:1], k), 0.1)
166os.makedirs("out/probe", exist_ok=True)
167torch.save({"W": W, "b": b, "mean": mean, "spread": spread, "names": people}, f"out/probe/due-{meta['name']}.pt")
168
169new, seen = torch.tensor(result["new"]["accuracy"]).mean(0), torch.tensor(result["seen"]["accuracy"]).mean(0)
170t, e = TARGETS.index, ROLES.index("end")
171show = lambda row: " ".join(f"{100 * v:4.0f}" for v in row.tolist())
172print(f"\nthe due name at the last position, percent right (chance {100 / k:.0f}), mean of {patterns} held-out patterns")
173print("  after block             emb    1    2    3    4    5    6    7    8")
174for v, version in enumerate(VERSIONS):
175    print(f"  reader, new pairs, {version:8s}" + show(new[v, :, e, t("due")]))
176for v, version in enumerate(VERSIONS):
177    print(f"  lens, {version:8s}             " + show(torch.tensor(result["new"]["lens"]).mean(0)[v]))
178for v, version in enumerate(VERSIONS):
179    print(f"  combined readers, {version:8s} " + show(torch.tensor(result["new"]["composite"]).mean(0)[v, :, e]))
180for v, version in enumerate(VERSIONS):
181    print(f"  seen pairs: due, {version:8s}  " + show(seen[v, :, e, t("due")]))
182print("  seen pairs: control           " + show(seen[:, :, e, t("control")].mean(0)))
183print("  new pairs: control            " + show(new[:, :, e, t("control")].mean(0)))
184print("\nthe worse of the two versions, new pairs, by depth (rows) and position (columns: " + ", ".join(ROLES) + ")")
185for target in TARGETS[:4]:
186    print(f"  {target}")
187    worse = new[:, :, :, t(target)].min(0).values
188    for d in range(depth):
189        print(f"    {'emb' if d == 0 else d:>3}  " + show(worse[d]))
190print("\nfrom the tokens' identities alone, no model:", result["tokens only"])