minterp

scripts/limits.py

607 lines · View on GitHub · Linked from 8. Limits

1"""
2Step 8: how much of the seven steps holds elsewhere? See docs/08-limits.md.
3
4The notes so far reported two story models that differ in the seed of
5their training, one dictionary per model, and no other model. This script
6measures nothing on the behavior: it reads the records of the earlier
7steps and sets them side by side.
8
9models         the headline number of each step for the two story models,
10               and for the open model wherever a script ran on it
11               (scripts/behavior.py, lens.py and patch.py with --model
12               qwen, and scripts/open.py)
13dictionaries   steps 5 to 7 again with a second dictionary for each story
14               model, trained in the same way from another seed
15               (scripts/sae.py, features.py and steer.py with --dict s1)
16matches        which latents of the first dictionary have a counterpart
17               in the second: by the cosine of their decoder directions,
18               and for the latents the earlier steps named, by whether the
19               counterpart plays the same part. This table loads the
20               models, to see where two latents fire.
21
22Two columns are called the same by one rule, applied to every row: a
23number when the two differ by at most a fifth of the larger (or by 0.05),
24a count or a name when they are equal.
25
26A third seed of the story model (seed2) and a third dictionary for the
27first two models (--seed 2 --dict s2) are read by the same rule, which was
28written here before either had been run. For three columns the rule is
29applied to each of the three pairs, since it does not carry over: a can
30be the same as b and b as c without a being the same as c. A row is the
31same in all three when all three pairs are. Otherwise the record says
32which of the first two the third agrees with. Reported per table: the
33rows the same in all three; of the rows the first two agreed on, how many
34the third joins; of the rows they differed on, whether the third sides
35with the first, the second, both or neither. The third model's dictionary
36is trained after the block of its own head at the acting name, which is
37how block 4 was chosen for the first two.
38
39Settled after the third model's steps 1 to 3 had run and before their
40records were read: "its own head at the acting name" is the head with the
41largest share of the gap moved at that position alone (step 3's parts),
42whether or not one head or several pass 0.05 there. That gives block 4
43for the first two. If it gives another block for the third, the rows of
44steps 5 to 7 compare the models at different places, and no run is added
45to tell the seed from the place. The third model sorts the twelve names
46into the same six and six as the first two, so every count has the base
47its row names; the number of heads its circuit has is recorded beside the
48row on what they carry together. The agreement of each pair is counted
49too, since the rows the same in all three can only be fewer than the rows
50the first two agreed on. One model and one dictionary are added to it,
51and one dictionary to each of the first two; no row is run again. (Noted
52later: the six and six came from one forward pass per name made for this
53paragraph, when no record held the third model's kinds yet.)
54
55One look was added after the rows had been read, and is no row. Step 4's
56record of the third model has two heads at the last position that read
57the acting name and lower it, with shares of 0.62 and 0.45, where the
58first two models have one large one (0.87 against 0.13, 0.69 against
590.14). Step 6's two rows are about the larger. Written here before
60scripts/heads.py had been run on the third model: what the smaller, head
615 of block 6, adds to an acting "She" name against the other names of its
62kind on clean prompts is set beside the first two models' -0.26 and -0.23
63by the same rule, and reported whichever way it falls. The 24 rows and
64their tally stay as they are.
65
66An independent check of the note's section then measured two more things,
67and both are kept here as looks made after the check, with no comparison
68written beforehand and no row changed: step 6 on that second head
69(scripts/features.py --head 6.5), and a second and a third dictionary for
70the third model (--model seed2 --seed 1 --dict s1, and --seed 2 --dict
71s2) with steps 6 and 7 on each.
72
73    uv run scripts/limits.py
74"""
75
76import json
77import os
78
79import numpy as np
80import torch
81from minllm.data import load_tokens
82
83from minterp import sae as S
84from minterp import tasks
85from minterp.common import MINLLM, load, save
86from minterp.tap import Tap
87
88SEEDS, OPEN, TAG = ("seed0", "seed1"), "qwen", "s1"
89THIRD, TAG3 = "seed2", "s2"  # a third seed of the model, and a third dictionary for the first two
90KINDS = ("both She", "both He", "one of each")
91
92
93def record(folder, file, name, tag=""):
94    path = f"runs/{folder}/{file}-{name}{'-' + tag if tag else ''}.json"
95    if not os.path.exists(path):
96        return None
97    with open(path) as f:
98        return json.load(f)
99
100
101SPREAD = []  # what the last call of over() averaged: a row's value per sentence pattern, kept for the figure of the third seed
102
103
104def over(r, pick, key="templates"):
105    SPREAD[:] = [float(pick(t)) for t in r[key]]
106    return float(np.mean(SPREAD))
107
108
109def same(a, b, kind):
110    if a is None or b is None:
111        return None
112    if kind != "number":
113        return a == b
114    return abs(a - b) <= max(0.2 * max(abs(a), abs(b)), 0.05)
115
116
117def spread(value, name):
118    """A row's value per sentence pattern, where the row is a mean over the patterns; None where it is one number, a ratio of means or a count."""
119    SPREAD.clear()
120    value(name, "")
121    return list(SPREAD) or None
122
123
124def third(a, b, c, kind):
125    """The rule for three: every pair by the rule for two, with c the newcomer."""
126    if c is None:
127        return None
128    with_a, with_b = bool(same(a, c, kind)), bool(same(b, c, kind))
129    return {"agrees with": [with_a, with_b], "the same in all three": bool(same(a, b, kind)) and with_a and with_b}
130
131
132def tally(rows, first_two, verdict):
133    """Over the rows that have a third value: how the third sits with the first two."""
134    rows = [r for r in rows if verdict(r) is not None]
135    agreed, differed = [r for r in rows if first_two(r)], [r for r in rows if not first_two(r)]
136    sides = lambda r: {(True, True): "both", (True, False): "the first only", (False, True): "the second only", (False, False): "neither"}[tuple(verdict(r)["agrees with"])]
137    return {"rows": len(rows), "the same in all three": sum(verdict(r)["the same in all three"] for r in rows),
138            "by pair": {"the first and the second": len(agreed), "the first and the third": sum(verdict(r)["agrees with"][0] for r in rows),
139                        "the second and the third": sum(verdict(r)["agrees with"][1] for r in rows)},
140            "of the rows the first two agreed on": {"rows": len(agreed), "the third is the same as both": sum(verdict(r)["the same in all three"] for r in agreed)},
141            "of the rows they differed on": {"rows": len(differed), **{k: sum(sides(r) == k for r in differed) for k in ("the first only", "the second only", "both", "neither")}}}
142
143
144# ---------------------------------------------------------------- one number per step, per model
145def blocks_of(name):
146    return record("read", "behavior", name)["model"]["config"]["n_layers"]
147
148
149def behavior(version, sign):
150    def value(name, tag):
151        r = record("read", "behavior", name)
152        return over(r, lambda t: (sign * np.array(t[version]["logit_diff"]) > 0).mean())
153    return value
154
155
156def late_blocks(name, tag):
157    a = record("read", "lens", name)["attribution"]["clean"]
158    b = np.array(a["heads"]["mean"]).sum(1) + np.array(a["mlp"]["mean"])
159    return float(b[-len(b) // 4:].sum() / (b.sum() + a["embed"]["mean"]))
160
161
162def due_at_half_depth(version):
163    def value(name, tag):
164        r = record("probe", "probe", name)
165        if r is None:
166            return None
167        a = np.array(r["new"]["accuracy"])[:, r["versions"].index(version), len(r["new"]["accuracy"][0][0]) // 2, r["roles"].index("S"), r["targets"].index("due")]
168        return over({"templates": a.tolist()}, float)  # new pairs, mean over the held-out patterns
169    return value
170
171
172def leaves_at(name, tag, line=0.5):
173    """The last depth at which moving the acting name's position alone still carries `line` of the gap, as a share of the depth."""
174    r = record("patch", "patch", name)
175    at = np.mean([t["travel"]["carry"] for t in r["templates"]], 0)[:, r["travel groups"].index("S")]
176    return float(max(d for d, v in enumerate(at) if v >= line) / (len(at) - 1))
177
178
179def block_at_name(name):
180    """The block of the head with the largest share of the gap at the acting name: where a model's dictionary is trained."""
181    at = np.mean([t["parts"]["carry"]["S"] for t in record("patch", "patch", name)["templates"]], 0)[:, :-1]  # (blocks, heads), without the MLPs
182    return int(np.unravel_index(at.argmax(), at.shape)[0]) + 1
183
184
185def at_end_by_kind(name):
186    """Every head scripts/heads.py read at the last position: its share, and what it adds to an acting name against the uninvolved names of its kind, on clean prompts."""
187    r = record("open", "heads", name)
188    if r is None:
189        return None
190    key = lambda h: f"block {h['block']} head {h['head']}"
191    return [{"head": f"{h['block']}.{h['head']}", "carry": h["carry"],
192             "adds to an acting name against its own kind": {k: over(r, lambda t: t["by kind"][key(h)]["clean"]["acts"][k]) for k in ("She", "He")},
193             "reads the acting name with": over(r, lambda t: t["attention"][key(h)]["clean"]["S"])} for h in r["heads"] if h["acts at"] == "end"]
194
195
196def veto_head(name, tag):
197    r = record("circuit", "circuit", name)
198    if r is None:
199        return None
200    h = max((h for h in r["heads"] if h["acts at"] == "end"), key=lambda h: h["carry"])
201    return f"{h['block']}.{h['head']}"
202
203
204def circuit_sets(name, tag):
205    r = record("circuit", "circuit", name)
206    return None if r is None else over(r, lambda t: t["sets"]["the heads"])
207
208
209def sae(pick):
210    def value(name, tag):
211        r = record("sae", "sae", name, tag)
212        return None if r is None else float(pick(r))
213    return value
214
215
216def features(pick):
217    def value(name, tag):
218        r = record("features", "features", name, tag)
219        return None if r is None else pick(r)
220    return value
221
222
223def steered(direction, measure="share", group=None):
224    """Step 7's first experiment at dose 1: from runs/steer for a story model, from runs/open for the open one."""
225    def value(name, tag):
226        r = record("steer", "steer", name, tag)
227        T = r["who acts"]["templates"] if r else (record("open", "steer", name) or {}).get("templates")
228        if T is None or direction not in T[0]["steered"]:
229            return None
230        doses = (r or record("open", "steer", name))["doses"]
231        i = doses.index(1.0)
232        if group:
233            return float(np.mean([t["steered"][direction]["moved, in logits, by pair"][group][i] for t in T]) / np.mean([t["pairs"][group]["gap"] for t in T]))
234        return over({"templates": T}, lambda t: t["steered"][direction][measure][i])
235    return value
236
237
238def gap_ratio(name, tag):
239    r = record("steer", "steer", name, tag)
240    T = r["who acts"]["templates"] if r else (record("open", "steer", name) or {}).get("templates")
241    return None if T is None else float(np.mean([t["pairs"]["both She"]["gap"] for t in T]) / np.mean([t["pairs"]["one of each"]["gap"] for t in T]))
242
243
244def pronoun(direction, what, dose=1.0, groups=("He names", "She names")):
245    def value(name, tag):
246        r = record("steer", "steer", name, tag)
247        if r is None:
248            return None
249        i, T = r["doses"].index(dose), r["kind"]["templates"]
250        if what == "turned":
251            return sum(t["steered"][direction]["turned"][g][i] for t in T for g in groups)
252        if what == "moved by group":  # in logits, the mean of the first group
253            return float(abs(np.mean([t["steered"][direction]["gap"][groups[0]][i] for t in T]) - np.mean([t["plain"][groups[0]] for t in T])))
254        if what == "moved":  # in logits, the larger of the two groups
255            return float(max(abs(np.mean([t["steered"][direction]["gap"][g][i] for t in T]) - np.mean([t["plain"][g] for t in T])) for g in groups))
256        return float(np.mean([t["steered"][direction][what][groups[0]][i] for t in T]))
257    return value
258
259
260def kind_record(pick):
261    def value(name, tag):
262        r = record("steer", "steer", name, tag)
263        return None if r is None else pick(r["kind"])
264    return value
265
266
267def she_names_sharing(r):
268    she = r["names that go on with She"]
269    latents = [x["the head's latent"] for x in r["names"] if x["name"] in she]
270    return max(latents.count(j) for j in set(latents))
271
272
273def he_latents(r):
274    she = r["names that go on with She"]
275    return len({x["the head's latent"] for x in r["names"] if x["name"] not in she})
276
277
278T = lambda r: r["templates"]
279MODELS = [  # step, what, kind of value, how to get it
280    (1, "clean prompts with B above A, share", "number", behavior("clean", 1)),
281    (1, "flipped prompts with A above B, share", "number", behavior("flipped", -1)),
282    (1, "share of the clean logit difference written by the last quarter of the blocks", "number", late_blocks),
283    (2, "the due name read at the acting name at half depth, on new pairs, clean prompts, share right", "number", due_at_half_depth("clean")),
284    (2, "the same on flipped prompts", "number", due_at_half_depth("flipped")),
285    (3, "the last depth at which the acting name alone carries half of the gap, as a share of the depth", "number", leaves_at),
286    (4, "the head that carries most at the last position (block.head)", "name", veto_head),
287    (4, "the seven heads of the circuit together carry, share of the gap", "number", circuit_sets),
288    (5, "variance the dictionary misses on stories, share", "number", sae(lambda r: r["rebuilds"]["unexplained variance"])),
289    (5, "the loss rises by, share", "number", sae(lambda r: r["costs"]["loss rises by"])),
290    (6, "who acts carried by what the dictionary rebuilds, share of the gap", "number", features(lambda r: over(r, lambda t: t["parts"]["rebuilt"]))),
291    (6, "carried by the highest-ranked latent", "number", features(lambda r: over(r, lambda t: t["by count"]["real"][0]))),
292    (6, "of the six She names, how many share the veto head's latent", "count", features(she_names_sharing)),
293    (6, "the veto head's write to an acting She name against its own kind, logits", "number",
294     features(lambda r: over(r, lambda t: t["head"]["by kind of the acting name"]["She"]["against its own kind"]))),
295    (7, "the pair direction at dose 1, share of the way", "number", steered("pair")),
296    (7, "its part with a term per name", "number", steered("names")),
297    (7, "the names' means with the vector every flipped prompt shares", "number", steered("means + order")),
298    (7, "the names' means alone", "number", steered("means")),
299    (7, "the shared vector alone", "number", steered("order")),
300    (7, "a random direction as long as the pair direction", "number", steered("random, long")),
301    (7, "the kind alone, on pairs of two kinds, share of their gap", "number", steered("kind", group="one of each")),
302    (7, "the gap between two She names against the gap between names of two kinds", "number", gap_ratio),
303    (7, "pronouns turned by the difference of means at dose 1, of 72", "count", pronoun("means", "turned")),
304    (7, "pronouns turned by the shared latent at its mean value, of 72", "count", pronoun("latent", "turned")),
305]
306DICTIONARIES = [
307    (5, "variance the dictionary misses on stories, share", "number", sae(lambda r: r["rebuilds"]["unexplained variance"])),
308    (5, "the loss rises by, share", "number", sae(lambda r: r["costs"]["loss rises by"])),
309    (6, "who acts carried by what the dictionary rebuilds", "number", features(lambda r: over(r, lambda t: t["parts"]["rebuilt"]))),
310    (6, "carried by the highest-ranked latent", "number", features(lambda r: over(r, lambda t: t["by count"]["real"][0]))),
311    (6, "by the eight highest", "number", features(lambda r: over(r, lambda t: t["by count"]["real"][3]))),
312    (6, "by the 32 highest", "number", features(lambda r: over(r, lambda t: t["by count"]["real"][5]))),
313    (6, "the gradient's estimate against the real effect, correlation", "number", features(lambda r: over(r, lambda t: t["one latent"]["correlation"]))),
314    (6, "the two names' latents moved for real, share of the gap", "number", features(lambda r: over(r, lambda t: t["names"]["the head's latents"]["moved for real"]))),
315    (6, "of the six She names, how many share the veto head's latent", "count", features(she_names_sharing)),
316    (6, "different latents among the six He names", "count", features(he_latents)),
317    (6, "the shared latent off: the acting She name against the nine, logits", "number",
318     features(lambda r: over(r, lambda t: t["head"]["the shared latent switched off, where a She name acts"]["the acting name against the nine"]))),
319    (6, "the same against its own kind", "number",
320     features(lambda r: over(r, lambda t: t["head"]["the shared latent switched off, where a She name acts"]["the acting name against its own kind"]))),
321    (7, "the two names' latents steered, share of the way", "number", steered("latents")),
322    (7, "the shared latent at its mean value moves the pronoun measure after a He name by, logits (mean)", "number", pronoun("latent", "moved by group")),
323    (7, "pronouns it turns, of 72", "count", pronoun("latent", "turned")),
324    (7, "of them after the 36 He names", "count", pronoun("latent", "turned", groups=("He names",))),
325    (7, "and after the 36 She names", "count", pronoun("latent", "turned", groups=("She names",))),
326    (7, "at twice that value, of 72", "count", pronoun("latent", "turned", dose=2.0)),
327    (7, "the rebuilt part of the difference of means turns, of 72", "count", pronoun("rebuilt", "turned")),
328    (7, "the missed part turns, of 72", "count", pronoun("missed", "turned")),
329    (7, "the missed part moves the measure by, logits (the larger of the two groups' means)", "number", pronoun("missed", "moved")),
330    (7, "the dictionary reads the latent back at, share of what was written", "number",
331     pronoun("latent", "the shared latent as the dictionary reads it, against its usual value")),
332    (7, "its encoder's reading of one unit of the latent's direction", "number", kind_record(lambda k: k["the encoder's reading of one unit of the latent's direction"])),
333    (7, "where the reading starts on a He name, share of the latent's mean value", "number", kind_record(lambda k: k["the reading before the cut on He names, against the usual value"])),
334    (7, "latents that differ by more than 0.5 between the kinds (mean of six sentences)", "number",
335     kind_record(lambda k: float(np.mean([t["latents that differ by more than half a unit between the kinds"] for t in k["templates"]])))),
336]
337
338result = {"rule": "a number is the same when two columns differ by at most a fifth of the larger, or by 0.05; a count or a name when they are equal",
339          "models": {"columns": [*SEEDS, OPEN], "rows": []}, "dictionaries": {"columns": [f"{s}, {d}" for s in SEEDS for d in ("first", "second")], "rows": []}}
340second = 0
341for step, what, kind, value in MODELS:
342    values = [value(name, "") for name in (*SEEDS, OPEN)]
343    result["models"]["rows"].append({"step": step, "what": what, "kind": kind, "values": values, "the two story models agree": same(values[0], values[1], kind),
344                                     "the open model agrees with": None if values[2] is None else [bool(same(values[0], values[2], kind)), bool(same(values[1], values[2], kind))]})
345    second += bool(same(value(SEEDS[0], TAG), value(SEEDS[1], TAG), kind))  # the same row with the second dictionaries, where a row has one
346result["models"]["rows the two story models agree on"] = {"with the first dictionaries": sum(bool(r["the two story models agree"]) for r in result["models"]["rows"]),
347                                                          "with the second dictionaries": second, "of": len(MODELS)}
348# The depth row depends on the line drawn in the open model. With other lines:
349result["models"]["the depth row with another line"] = {str(line): [leaves_at(name, "", line) for name in (*SEEDS, OPEN)] for line in (0.9, 0.5, 0.25, 0.1)}
350# The vector every flipped prompt shares, by depth (scripts/open.py on all three models): the names' means alone, the vector alone, the two together.
351result["models"]["by depth"] = {}
352for name in (*SEEDS, OPEN):
353    r = record("open", "steer", name)
354    if r is not None:
355        result["models"]["by depth"][name] = {"depths": r["depths"], **{d: np.mean([t["depth"][d]["share"] for t in r["templates"]], 0).tolist() for d in ("pair", "means", "order", "means + order")}}
356for step, what, kind, value in DICTIONARIES:
357    values = [value(name, tag) for name in SEEDS for tag in ("", TAG)]
358    result["dictionaries"]["rows"].append({"step": step, "what": what, "kind": kind, "values": values,
359                                           "the two dictionaries agree": [same(values[0], values[1], kind), same(values[2], values[3], kind)]})
360
361# ---------------------------------------------------------------- a third seed, and a third dictionary
362# The third model's records, where every step has run on it: the same 24 rows, by the rule for three.
363needed = (("read", "behavior"), ("read", "lens"), ("probe", "probe"), ("patch", "patch"), ("circuit", "circuit"), ("sae", "sae"), ("features", "features"), ("steer", "steer"))
364if all(record(folder, file, THIRD) is not None for folder, file in needed):
365    rows = []
366    for (step, what, kind, value), row in zip(MODELS, result["models"]["rows"]):
367        v = value(THIRD, "")
368        rows.append({"step": step, "what": what, "kind": kind, "value": v, **third(row["values"][0], row["values"][1], v, kind), "by pattern": [spread(value, s) for s in (*SEEDS, THIRD)]})
369    paired = list(zip(result["models"]["rows"], rows))
370    result["models"]["a third seed"] = {
371        "name": THIRD, "the block of its dictionary": record("sae", "sae", THIRD)["block"], "the block of the head with the largest share at the acting name": block_at_name(THIRD),
372        "the same block for the first two": [block_at_name(s) for s in SEEDS], "heads of the circuit": [len(record("circuit", "circuit", s)["heads"]) for s in (*SEEDS, THIRD)],
373        "names that go on with She": record("features", "features", THIRD)["names that go on with She"],
374        "rows": rows, "tally": tally(paired, lambda p: p[0]["the two story models agree"], lambda p: p[1]),
375        "the depth row with another line": {str(line): leaves_at(THIRD, "", line) for line in (0.9, 0.5, 0.25, 0.1)},
376    }
377    # The look added after the rows were read: every head at the last position by kind, where scripts/heads.py has run.
378    she_row = features(lambda r: over(r, lambda t: t["head"]["by kind of the acting name"]["She"]["against its own kind"]))
379    at_end = {s: at_end_by_kind(s) for s in (*SEEDS, THIRD)}
380    for h in at_end[THIRD] or []:
381        h["an acting She name, beside the first two models' veto heads"] = third(she_row(SEEDS[0], ""), she_row(SEEDS[1], ""), h["adds to an acting name against its own kind"]["She"], "number")
382    result["models"]["a third seed"]["the heads at the last position, by kind"] = at_end
383    # Looks made after an independent check of the section, and no rows: step 6 on the third model's second veto head,
384    # and steps 6 and 7 on two more dictionaries for the third model.
385    measures = ("against the nine", "against its own kind", "its own kind against the other")
386
387    def step6(name, tag):
388        f = record("features", "features", name, tag)
389        return None if f is None else {
390            "head": f["veto head"], "She names that share its first choice": she_names_sharing(f),
391            "adds where a name of each kind acts": {k: {m: over(f, lambda t: t["head"]["by kind of the acting name"][k][m]) for m in measures} for k in ("She", "He")}}
392
393    def together(heads_):
394        """The two heads with the largest share among those that give the acting name more than half of their attention."""
395        two = sorted((h for h in heads_ if h["reads the acting name with"] > 0.5), key=lambda h: -h["carry"])[:2]
396        return {"heads": [h["head"] for h in two], **{k: sum(h["adds to an acting name against its own kind"][k] for h in two) for k in ("She", "He")}}
397
398    by_block = lambda name: np.mean([t["travel"]["carry"] for t in record("patch", "patch", name)["templates"]], 0)[:, record("patch", "patch", name)["travel groups"].index("S")].tolist()
399    after = {"by dictionary": {}, "the first two models' veto heads": {s: step6(s, "") for s in SEEDS},
400             "the two largest heads that read the acting name, together, against its own kind": {s: together(at_end[s]) for s in (*SEEDS, THIRD) if at_end[s]},
401             "the acting name's position moved alone, by block": {s: by_block(s) for s in (*SEEDS, THIRD)}}
402    # The second head is the other of the third model's two, found above and not named here.
403    picked = record("features", "features", THIRD)["veto head"]
404    other = next((h for h in together(at_end[THIRD] or [])["heads"] if "block {} head {}".format(*h.split(".")) != picked), None)
405    for label, tag in (("first", ""), ("second", "s1"), ("third", "s2")):
406        if record("features", "features", THIRD, tag) is not None:
407            after["by dictionary"][label] = {"the head step 6 picks": step6(THIRD, tag), "the second head": other and step6(THIRD, f"{tag}-h{other}" if tag else f"h{other}"),
408                                             "pronouns the shared latent turns, of 72": pronoun("latent", "turned")(THIRD, tag)}
409    result["models"]["a third seed"]["looks after the check"] = after
410    r = record("open", "steer", THIRD)
411    if r is not None:
412        result["models"]["by depth"][THIRD] = {"depths": r["depths"], **{d: np.mean([t["depth"][d]["share"] for t in r["templates"]], 0).tolist() for d in ("pair", "means", "order", "means + order")}}
413# A third dictionary for each of the first two models: the same 25 rows, by the rule for three.
414if all(record(step, step, seed, TAG3) is not None for seed in SEEDS for step in ("sae", "features", "steer")):
415    rows = []
416    for (step, what, kind, value), row in zip(DICTIONARIES, result["dictionaries"]["rows"]):
417        values = [value(seed, TAG3) for seed in SEEDS]
418        rows.append({"step": step, "what": what, "kind": kind, "values": values,
419                     "by model": [third(row["values"][2 * i], row["values"][2 * i + 1], values[i], kind) for i in range(len(SEEDS))]})
420    paired = list(zip(result["dictionaries"]["rows"], rows))
421    result["dictionaries"]["a third dictionary"] = {
422        "columns": [f"{s}, third" for s in SEEDS], "rows": rows,
423        "tally": {seed: tally(paired, lambda p, i=i: p[0]["the two dictionaries agree"][i], lambda p, i=i: p[1]["by model"][i]) for i, seed in enumerate(SEEDS)},
424    }
425
426# ---------------------------------------------------------------- which latents have a counterpart
427def one_to_one(c, floor=0.7):
428    """A greedy one-to-one matching of rows to columns by cosine, highest first. Pairs under the floor are left unmatched: -1."""
429    rows, cols = (c >= floor).nonzero(as_tuple=True)
430    partner, taken = torch.full((c.shape[0],), -1), torch.zeros(c.shape[1], dtype=torch.bool)
431    for k in c[rows, cols].argsort(descending=True).tolist():
432        i, j = rows[k].item(), cols[k].item()
433        if partner[i] < 0 and not taken[j]:
434            partner[i], taken[j] = j, True
435    return partner
436
437
438unit = lambda v: v / v.norm(dim=1, keepdim=True)
439shares = lambda best: {"at a cosine of 0.9 or more": (best >= 0.9).float().mean().item(), "at 0.7 or more": (best >= 0.7).float().mean().item(),
440                       "at 0.5 or more": (best >= 0.5).float().mean().item(), "median cosine": best.median().item()}
441result["matches"] = {}
442for seed in SEEDS:
443    model, tok, meta = load(seed)
444    device = next(model.parameters()).device
445    one, two = S.load(S.path(seed, 4), "cpu"), S.load(S.path(seed, 4, TAG), "cpu")
446    cosine = one.dec.detach() @ two.dec.detach().T  # (width, width): both sets of directions have length one
447    best = cosine.max(1)
448    inside = one.dec.detach() @ one.dec.detach().T
449    inside.fill_diagonal_(-1)
450    g = torch.Generator().manual_seed(0)
451    noise = unit(torch.randn(one.width, one.dec.shape[1], generator=g)) @ unit(torch.randn(two.width, two.dec.shape[1], generator=g)).T
452    # The stricter measure of Paulo and Belrose: one-to-one, the same partner by the encoder and by the decoder, 0.7 or more in both.
453    by_dec, by_enc = one_to_one(cosine), one_to_one(unit(one.enc.detach().T) @ unit(two.enc.detach().T).T)
454    # How often each latent of the first dictionary fires on held-out stories.
455    SEQ = model.cfg.max_seq_len
456    data = load_tokens(os.path.join(MINLLM, "data/tinystories/valid.bin"))
457    starts = np.random.default_rng(2).integers(0, len(data) - SEQ - 1, 300)
458    fires, seen = torch.zeros(one.width), 0
459    for i in range(0, 300, 100):
460        idx = torch.from_numpy(np.stack([data[k:k + SEQ].astype(np.int64) for k in starts[i:i + 100]])).to(device)
461        with torch.no_grad(), Tap(model, keep=("resid",)) as tap:
462            model(idx)
463        x = tap["resid", 4].float().reshape(-1, one.dec.shape[1]).cpu()
464        with torch.no_grad():
465            fires += (one.encode(x) > 0).float().sum(0)
466        seen += x.shape[0]
467    rate = fires / seen
468    bins = {"fires on up to 0.1% of story tokens": rate <= 1e-3, "on 0.1% to 1%": (rate > 1e-3) & (rate <= 1e-2), "on more than 1%": rate > 1e-2}
469
470    f1, f2 = record("features", "features", seed), record("features", "features", seed, TAG)
471    she = f1["names that go on with She"]
472    head1, head2 = ({x["name"]: x["the head's latent"] for x in f["names"]} for f in (f1, f2))
473    step5_1, step5_2 = ({x["name"]: x["step 5's latent"] for x in f["names"]} for f in (f1, f2))
474    shared1, shared2 = f1["the shared latent"], f2["the shared latent"]
475    # Where the two shared latents fire: at the acting name of every clean and flipped prompt.
476    people = tasks.names(tok)
477    streams, acting = [], []
478    for batch in tasks.batches(tok, device, people=people):
479        for version, col in (("clean", 0), ("flipped", 1)):
480            with torch.no_grad(), Tap(model, keep=("resid",)) as tap:
481                model(getattr(batch, version))
482            streams.append(tap["resid", 4][:, batch.pos["S"]].float().cpu())
483            acting += [t[col] in she for t in batch.names]
484    streams, acting = torch.cat(streams), torch.tensor(acting)
485    with torch.no_grad():
486        z1, z2 = one.encode(streams)[:, shared1], two.encode(streams)[:, shared2]
487    fire = lambda a, b: torch.corrcoef(torch.stack([a, b]))[0, 1].item()
488
489    def by_latent(table1, table2):
490        """One entry per latent of the first dictionary that a table names: its best counterpart, and whether the second dictionary's table gives that counterpart to most of the same names."""
491        out = []
492        for latent in sorted(set(table1.values())):
493            names = [p for p in people if table1[p] == latent]
494            j = best.indices[latent].item()
495            out.append({"latent": latent, "names": names, "cosine of the best counterpart": best.values[latent].item(),
496                        "the counterpart is the second dictionary's latent for most of these names": sum(table2[p] == j for p in names) * 2 > len(names)})
497        return out
498
499    result["matches"][seed] = {
500        "latents of the first dictionary": one.width, "latents that never fire on the stories": int((rate == 0).sum()), "story tokens": seen,
501        "with a counterpart in the second dictionary": shares(best.values),
502        "by how often a latent fires": {label: {"latents": int(m.sum()), **shares(best.values[m])} for label, m in bins.items()},
503        "for comparison": {"the nearest other latent of the same dictionary": shares(inside.max(1).values), "two sets of random directions": shares(noise.max(1).values)},
504        "one to one": {"at a cosine of 0.7 or more by the decoder": (by_dec >= 0).float().mean().item(),
505                       "the same partner by encoder and decoder, 0.7 or more in both": ((by_dec >= 0) & (by_dec == by_enc)).float().mean().item()},
506        "the shared latent": {"first": shared1, "second": shared2, "cosine of the best counterpart": best.values[shared1].item(),
507                              "the best counterpart is the second dictionary's shared latent": bool(best.indices[shared1].item() == shared2),
508                              "correlation of the two over all prompts": fire(z1, z2), "among the prompts where a She name acts": fire(z1[acting], z2[acting]),
509                              "share of the other prompts on which each fires": [(z1[~acting] > 0).float().mean().item(), (z2[~acting] > 0).float().mean().item()],
510                              "share of the She prompts on which each fires": [(z1[acting] > 0).float().mean().item(), (z2[acting] > 0).float().mean().item()], "prompts": len(acting)},
511        "the veto head's latents": by_latent(head1, head2), "step 5's latents": by_latent(step5_1, step5_2),
512    }
513    # A third dictionary, where one was trained: counterparts of the first and of the second in it, and latents of the first found in both others.
514    f3 = record("features", "features", seed, TAG3)
515    if f3 is not None and os.path.exists(S.path(seed, 4, TAG3)):
516        three = S.load(S.path(seed, 4, TAG3), "cpu")
517        in_third, second_in_third = (d.dec.detach() @ three.dec.detach().T for d in (one, two))
518        b13, b23 = in_third.max(1), second_in_third.max(1)
519        shared3 = f3["the shared latent"]
520        result["matches"][seed]["a third dictionary"] = {
521            "the first dictionary's latents with a counterpart in the third": shares(b13.values),
522            "the second dictionary's latents with a counterpart in the third": shares(b23.values),
523            "latents of the first with a counterpart at 0.7 or more in the second and in the third": ((best.values >= 0.7) & (b13.values >= 0.7)).float().mean().item(),
524            "of those that fire on more than 1% of story tokens": ((best.values >= 0.7) & (b13.values >= 0.7))[rate > 1e-2].float().mean().item(),
525            "the shared latent": {"third": shared3,
526                                  "cosine of the first's best counterpart": b13.values[shared1].item(), "which is the third's shared latent": bool(b13.indices[shared1].item() == shared3),
527                                  "cosine of the second's best counterpart": b23.values[shared2].item(), "which is the third's shared latent, from the second": bool(b23.indices[shared2].item() == shared3)},
528        }
529
530# ---------------------------------------------------------------- print
531def show(v):
532    return "     —" if v is None else f"{v:>6}" if isinstance(v, (str, int)) and not isinstance(v, bool) else f"{v:6.2f}"
533
534
535mark = {True: "same", False: "differs", None: ""}
536to_both = lambda a: "" if a is None else {(True, True): "same as both", (True, False): "as seed0 only", (False, True): "as seed1 only", (False, False): "as neither"}[tuple(a)]
537print(result["rule"] + "\n")
538print(f"{'':4}{'':98} {'seed0':>6} {'seed1':>6} {'open':>6}   the seeds   the open model")
539for row in result["models"]["rows"]:
540    print(f"{row['step']:>2}  {row['what'][:98]:98} " + " ".join(show(v) for v in row["values"]) + f"   {mark[row['the two story models agree']]:9}   {to_both(row['the open model agrees with'])}")
541print("the two story models agree on:", result["models"]["rows the two story models agree on"])
542print("the depth row with another line (seed0, seed1, open):", {k: [round(x, 2) for x in v] for k, v in result["models"]["the depth row with another line"].items()})
543for name, d in result["models"]["by depth"].items():
544    print(f"{name} by depth {d['depths']}: " + "; ".join(f"{k} " + " ".join(f"{x:.2f}" for x in d[k]) for k in ("means", "order", "means + order")))
545print(f"\n{'':4}{'':90} {'seed0':>13} {'seed1':>13}")
546print(f"{'':4}{'':90} {'first':>6} {'second':>6} {'first':>6} {'second':>6}")
547for row in result["dictionaries"]["rows"]:
548    print(f"{row['step']:>2}  {row['what'][:90]:90} " + " ".join(show(v) for v in row["values"]) + "   " + " / ".join(mark[a] for a in row["the two dictionaries agree"]))
549pct = lambda d: f"{100 * d['at a cosine of 0.9 or more']:.0f}% / {100 * d['at 0.7 or more']:.0f}% / {100 * d['at 0.5 or more']:.0f}%, median {d['median cosine']:.2f}"
550for seed, m in result["matches"].items():
551    s_ = m["the shared latent"]
552    print(f"\n{seed}: of {m['latents of the first dictionary']} latents ({m['latents that never fire on the stories']} never fire), with a counterpart at 0.9 / 0.7 / 0.5: {pct(m['with a counterpart in the second dictionary'])}")
553    for label, d in m["by how often a latent fires"].items():
554        print(f"  {label} ({d['latents']}): {pct(d)}")
555    for label, d in m["for comparison"].items():
556        print(f"  for comparison, {label}: {pct(d)}")
557    print("  one to one: " + ", ".join(f"{k} {100 * v:.1f}%" for k, v in m["one to one"].items()))
558    print(f"  the shared latent {s_['first']} -> {s_['second']}: cosine {s_['cosine of the best counterpart']:.2f}, mutual: {s_['the best counterpart is the second dictionary' + chr(39) + 's shared latent']}; "
559          f"correlation over all {s_['prompts']} prompts {s_['correlation of the two over all prompts']:.2f}, among those where a She name acts {s_['among the prompts where a She name acts']:.2f}; "
560          f"fires on {[round(x, 2) for x in s_['share of the She prompts on which each fires']]} of those and {[round(x, 2) for x in s_['share of the other prompts on which each fires']]} of the rest")
561    for table in ("the veto head's latents", "step 5's latents"):
562        hit = "the counterpart is the second dictionary's latent for most of these names"
563        rows_ = m[table]
564        print(f"  {table}: {len(rows_)} latents; the counterpart is the second's for the same names for {sum(x[hit] for x in rows_)}; at a cosine of 0.7 or more {sum(x['cosine of the best counterpart'] >= 0.7 for x in rows_)}; "
565              + " ".join(f"{'+'.join(x['names'])[:14]} {x['cosine of the best counterpart']:.2f}{'*' if x[hit] else ''}" for x in rows_))
566
567sides = lambda v: "" if v is None else "same in all three" if v["the same in all three"] else {(True, True): "as each, which differ", (True, False): "as the first only", (False, True): "as the second only", (False, False): "as neither"}[tuple(v["agrees with"])]
568t3 = result["models"].get("a third seed")
569if t3:
570    print(f"\na third seed ({t3['name']}, dictionary after block {t3['the block of its dictionary']}; She names: {', '.join(t3['names that go on with She'])})")
571    for row, new in zip(result["models"]["rows"], t3["rows"]):
572        print(f"{row['step']:>2}  {row['what'][:98]:98} " + " ".join(show(v) for v in (*row["values"][:2], new["value"])) + f"   {sides(new)}")
573    print("  ", t3["tally"])
574    print("   the depth row with another line:", {k: round(v, 2) for k, v in t3["the depth row with another line"].items()})
575    for s, heads_ in t3["the heads at the last position, by kind"].items():
576        for h in heads_ or []:
577            k = h["adds to an acting name against its own kind"]
578            print(f"   {s} head {h['head']}: share {h['carry']:.2f}, reads the acting name with {h['reads the acting name with']:.2f}, adds to an acting name against its own kind: She {k['She']:.2f}, He {k['He']:.2f}"
579                  + (f"   {sides(h[chr(34)[:0] + 'an acting She name, beside the first two models' + chr(39) + ' veto heads'])}" if s == THIRD else ""))
580if t3:
581    a3 = t3["looks after the check"]
582    kinds_ = lambda h: "  ".join(f"{k}: " + " / ".join(f"{h['adds where a name of each kind acts'][k][m]:.2f}" for m in ("against the nine", "against its own kind", "its own kind against the other")) for k in ("She", "He"))
583    print("   looks after the check (adds: against the nine / its own kind / its own kind against the other)")
584    for s, h in a3["the first two models' veto heads"].items():
585        print(f"   {s} {h['head']}: first choice shared by {h['She names that share its first choice']} She names; {kinds_(h)}")
586    for label, d in a3["by dictionary"].items():
587        for which in ("the head step 6 picks", "the second head"):
588            if d[which]:
589                print(f"   {t3['name']}, {label} dictionary, {which} ({d[which]['head']}): first choice shared by {d[which]['She names that share its first choice']} She names; {kinds_(d[which])}")
590        print(f"   {t3['name']}, {label} dictionary: the shared latent turns {d['pronouns the shared latent turns, of 72']} of 72 pronouns")
591    print("   together:", {s: {k: (v if k == "heads" else round(v, 2)) for k, v in d.items()} for s, d in a3["the two largest heads that read the acting name, together, against its own kind"].items()})
592    print("   the acting name's position moved alone, by block:", {s: [round(x, 2) for x in v] for s, v in a3["the acting name's position moved alone, by block"].items()})
593d3 = result["dictionaries"].get("a third dictionary")
594if d3:
595    print(f"\na third dictionary\n{'':4}{'':90} {'seed0':>20} {'seed1':>20}")
596    for row, new in zip(result["dictionaries"]["rows"], d3["rows"]):
597        v = row["values"]
598        print(f"{row['step']:>2}  {row['what'][:90]:90} " + " ".join(show(x) for x in (v[0], v[1], new["values"][0], v[2], v[3], new["values"][1])) + "   " + " / ".join(sides(x) for x in new["by model"]))
599    for seed, t in d3["tally"].items():
600        print(f"   {seed}:", t)
601    for seed, m in result["matches"].items():
602        if "a third dictionary" in m:
603            a = m["a third dictionary"]
604            print(f"   {seed}: the first in the third: {pct(a['the first dictionary' + chr(39) + 's latents with a counterpart in the third'])}; the second in the third: {pct(a['the second dictionary' + chr(39) + 's latents with a counterpart in the third'])}; "
605                  f"of the first, in both others at 0.7 or more: {100 * a['latents of the first with a counterpart at 0.7 or more in the second and in the third']:.0f}%; the shared latent: {a['the shared latent']}")
606
607save("runs/limits/limits.json", result)