tests/test_stack.py
102 lines · View on GitHub · Linked from 9. The practitioner's stack
1"""
2What docs/09-stack.md claims about the one library that was run, checked
3against runs/stack/. The record was made with the library laid over the
4project (`uv run --with transformer-lens==4.4.0 --with transformers==5.24.0
5scripts/stack.py`), so these tests need neither the library nor the open
6model. What the note says of documentation cannot be tested here.
7"""
8
9import json
10
11
12def run():
13 with open("runs/stack/stack-qwen.json") as f:
14 return json.load(f)
15
16
17def test_the_versions_and_the_prompts_the_note_names():
18 r = run()
19 assert r["versions"] == {"transformer-lens": "4.4.0", "transformers": "5.24.0", "torch": "2.15.0"}
20 assert (r["prompts"]["count"], r["prompts"]["of"], r["prompts"]["first names"]) == (16, 132, {"Lily": 11, "Tom": 5})
21 assert r["the tokenizers decode the prompt alike"] and r["a string gets no first token from the library"]
22
23 assert r["module paths of the transformers model"] == {"model.layers": 24, "lm_head": True, "transformer.h": False}
24
25
26def test_three_comparisons_of_logits_and_what_wrapping_changes():
27 r = run()
28 logits = r["logits"]
29 assert round(logits["largest entry"], 1) == 21.4
30 three = [logits[k] for k in ("minllm's loader against transformers, before wrapping", "the library against minterp", "the library against transformers, before wrapping")]
31 assert all(round(v, 5) == 0.00008 for v in three) and len(set(three)) == 3
32
33 assert r["calls of transformers' own modules in one pass"] == {"before wrapping": {"attention": 24, "MLP": 24}, "wrapped": {"attention": 0, "MLP": 24},
34 "wrapped, in compatibility mode": {"attention": 0, "MLP": 0}}
35
36
37def test_the_same_pieces_under_other_names():
38 r = run()
39 pieces = r["pieces"]
40 largest = lambda kind, key: max(v[key] for k, v in pieces.items() if k.startswith(f'("{kind}"'))
41 assert pieces['("resid", 0) against hook_embed'] == {"difference": 0, "largest entry": pieces['("resid", 0) against hook_embed']["largest entry"]}
42 assert round(pieces['("resid", 0) against hook_embed']["largest entry"], 2) == 0.07
43
44 assert round(pieces['("resid", 24) against blocks.23.hook_resid_post']["difference"], 4) == 0.0002 == round(largest("resid", "difference"), 4)
45 assert (round(pieces['("resid", 24) against blocks.23.hook_resid_post']["largest entry"]), round(pieces['("resid", 12) against blocks.11.hook_resid_post']["largest entry"])) == (40, 1394)
46 assert (round(r["sizes"]["the stream after the middle block, without the first position"]), round(r["sizes"]["attention weights"], 2)) == (12, 1.0)
47 assert round(max(largest("attn", "difference"), largest("mlp", "difference")), 4) == 0.0001 and round(max(largest("attn", "largest entry"), largest("mlp", "largest entry"))) == 63
48 assert round(largest("heads", "difference"), 5) == 0.00002 and round(largest("heads", "largest entry")) == 6
49 assert round(max(r["pattern"]), 6) == 0.000007 and len(pieces) == 13
50 assert r["names"] == {'("resid", 0)': "hook_embed", '("resid", i)': "blocks.i-1.hook_resid_post", '("attn", i)': "blocks.i-1.hook_attn_out",
51 '("mlp", i)': "blocks.i-1.hook_mlp_out", '("heads", i)': "blocks.i-1.attn.hook_z", "attention.patterns(model, tap)[:, i-1]": "blocks.i-1.attn.hook_pattern"}
52
53 assert r["canonical names"] == {"hook_embed": "embed.hook_out", "blocks.i.hook_resid_post": "blocks.i.hook_out", "blocks.i.hook_attn_out": "blocks.i.attn.hook_out",
54 "blocks.i.hook_mlp_out": "blocks.i.mlp.hook_out", "blocks.i.attn.hook_z": "blocks.i.attn.o.hook_in"}
55
56 seen = r["the name a hook function sees"]
57 asked = ("hook_embed", "blocks.11.hook_resid_post", "blocks.11.attn.hook_z", "blocks.11.attn.hook_pattern")
58 assert [seen[f"as wrapped: {name}"] for name in asked] == ["embed.hook_out", "blocks.11.hook_out", "blocks.11.attn.o.hook_in", "blocks.11.attn.hook_pattern"]
59 assert sum(seen[f"in compatibility mode: {name}"] == name and seen[f"as wrapped: {name}"] != name for name in asked) == 2
60
61
62def test_two_interventions_both_ways():
63 r = run()
64 p, s = r["patch"], r["steer"]
65 assert p["where"] == "after block 12, at the acting name"
66 assert (round(p["plain"], 3), round(p["by minterp"], 3), round(p["by the library"], 3)) == (6.552, -7.431, -7.431)
67 assert (round(s["by minterp"], 3), round(s["by the library"], 3)) == (0.087, 0.087)
68 assert p["logits, largest difference"] < 1e-3 and s["logits, largest difference"] < 1e-3
69
70
71def test_the_helper_that_needed_care():
72 a = run()["attribute"]
73 assert (a["parts"], a["parts of lens.attribute before the heads are summed"]) == (49, 361)
74 assert (round(a["the logit difference"], 3), round(a["minterp's parts sum to"], 3)) == (6.552, 6.552)
75 raw = a["the library's helper on the published weights"]
76 assert (round(raw["sums to"], 3), round(raw["largest difference to minterp, per part"], 2)) == (0.793, 5.04)
77 gain = a["the final norm's gain"]
78 assert (round(gain["smallest"], 2), round(gain["largest"], 1)) == (-0.01, 16.8)
79 by_hand, in_mode = a["with the stack multiplied by the gain first"], a["in compatibility mode"]
80 assert (round(by_hand["sums to"], 3), round(in_mode["sums to"], 3)) == (6.552, 6.552)
81 assert 0.00001 < by_hand["largest difference to minterp, per part"] < 0.00002 and 0.00001 < in_mode["largest difference to minterp, per part"] < 0.00002
82
83 assert (round(in_mode["for one token's logit it sums to"], 2), round(in_mode["that token's published logit"], 2)) == (22.48, 18.91)
84 assert abs(in_mode["for one token's logit it sums to"] - in_mode["the published logit less its mean over the vocabulary"]) < 1e-3
85
86
87def test_what_else_the_mode_changes():
88 m = run()["mode"]
89 assert round(m["logits, largest difference to the published model's"], 2) == 5.33 and m["log-probabilities, largest difference"] < 0.0001
90
91 assert m["the shift within one prompt and position, largest spread over the vocabulary"] < 0.0002
92 assert m["the shift against the mean of the published logits over the vocabulary"] < 0.0001
93
94 whole, last = m["the shift over prompts and positions"], m["at the last position"]
95 assert (round(whole["smallest"], 2), round(whole["largest"], 2)) == (-5.33, -0.25) and (round(last["smallest"], 2), round(last["largest"], 2)) == (-4.19, -3.06)
96
97 pieces = m["pieces after block 12 against minterp's, on the published weights"]
98 assert (round(pieces["heads"], 1), round(m["largest entry of the heads there"], 1)) == (1.3, 2.9)
99 assert pieces["attn"] < 0.0001 and pieces["mlp"] < 0.0001
100
101 change = m["the heads' change"]
102 assert change["largest spread over prompts and positions"] < 0.0001 and abs(change["largest entry"] - change["largest entry of that block's value bias"]) < 0.0001