tests/test_patch.py
57 lines · View on GitHub · Linked from 3. Patch
1import torch
2
3from minterp import patch, tasks
4from minterp.tap import head_writes, Tap
5
6
7def setup(model, tok, people):
8 batch = tasks.batches(tok, people=people)[0]
9 clean, clean_logits = patch.record(model, batch.clean)
10 flipped, flipped_logits = patch.record(model, batch.flipped)
11 return batch, clean, clean_logits, flipped, flipped_logits
12
13
14def test_the_whole_stream_carries_the_whole_answer(model, tok, people):
15 batch, clean, clean_logits, _, _ = setup(model, tok, people)
16 for depth in range(model.cfg.n_layers + 1):
17 out = patch.run(model, batch.flipped, {("resid", depth): patch.swap(clean["resid", depth])})
18 assert torch.allclose(out[:, -1], clean_logits[:, -1], atol=1e-5)
19
20
21def test_a_patch_where_the_runs_agree_changes_nothing(model, tok, people):
22 batch, clean, _, _, flipped_logits = setup(model, tok, people)
23 before = list(range(batch.pos["S"]))
24 for depth in range(model.cfg.n_layers + 1):
25 out = patch.run(model, batch.flipped, {("resid", depth): patch.swap(clean["resid", depth], positions=before)})
26 assert torch.allclose(out, flipped_logits, atol=1e-5)
27
28
29def test_a_patch_reaches_no_earlier_position(model, tok, people):
30 batch, clean, _, _, flipped_logits = setup(model, tok, people)
31 end = batch.pos["end"]
32 out = patch.run(model, batch.flipped, {("resid", 1): patch.swap(clean["resid", 1], positions=[end])})
33 assert torch.allclose(out[:, :end], flipped_logits[:, :end], atol=1e-5)
34 assert not torch.allclose(out[:, end], flipped_logits[:, end], atol=1e-4)
35
36
37def test_swapping_one_head_moves_only_that_heads_write(model, tok, people):
38 batch, clean, _, flipped, _ = setup(model, tok, people)
39 with Tap(model, edit={("heads", 2): patch.swap(clean["heads", 2], head=1)}) as tap:
40 model(batch.flipped)
41 writes = head_writes(model, tap, 2)
42 theirs = head_writes(model, type("S", (), {"__getitem__": lambda self, k: clean[k]})(), 2)
43 ours = head_writes(model, type("S", (), {"__getitem__": lambda self, k: flipped[k]})(), 2)
44 assert torch.allclose(writes[:, :, 1], theirs[:, :, 1], atol=1e-5)
45 for h in (0, 2, 3):
46 assert torch.allclose(writes[:, :, h], ours[:, :, h], atol=1e-5)
47
48
49def test_mean_zero_and_share():
50 a, b = {"k": torch.ones(2, 3, 4)}, {"k": 3 * torch.ones(2, 3, 4)}
51 assert patch.mean([a, b], "k").shape == (1, 3, 4) and (patch.mean([a, b], "k") == 2).all()
52 assert (patch.zero(a["k"]) == 0).all()
53 edit = patch.swap(patch.mean([a, b], "k"), positions=[1])
54 out = edit(a["k"])
55 assert (out[:, 1] == 2).all() and (out[:, 0] == 1).all() and (a["k"] == 1).all()
56 t, s = torch.tensor([0.0, 2.0]), torch.tensor([4.0, 6.0])
57 assert patch.share(t, t, s) == 0.0 and patch.share(s, t, s) == 1.0 and patch.share((t + s) / 2, t, s) == 0.5