tests/test_lens.py
32 lines · View on GitHub · Linked from 1. Read
1import torch
2
3from minterp import lens, tasks
4from minterp.tap import Tap
5
6
7def test_decoding_the_final_stream_gives_the_logits(model, tok, people):
8 batch = tasks.batches(tok, people=people)[0]
9 with Tap(model) as tap:
10 logits, _ = model(batch.clean)
11 assert torch.allclose(lens.decode(model, tap["resid", model.cfg.n_layers]), logits, atol=1e-5)
12
13
14def test_the_attribution_adds_up_to_the_logit_difference(model, tok, people):
15 for batch in tasks.batches(tok, people=people):
16 with Tap(model) as tap:
17 logits, _ = model(batch.clean)
18 parts = lens.attribute(model, tap, batch.right, batch.wrong)
19 n, cfg = len(batch), model.cfg
20 assert parts["embed"].shape == (n,)
21 assert parts["heads"].shape == (n, cfg.n_layers, cfg.n_heads) and parts["mlp"].shape == (n, cfg.n_layers)
22 total = parts["embed"] + parts["heads"].sum((1, 2)) + parts["mlp"].sum(1)
23 assert torch.allclose(total, tasks.logit_diff(logits, batch.right, batch.wrong), atol=1e-4)
24
25
26def test_the_attribution_of_a_position_other_than_the_last(model, tok, people):
27 batch = tasks.batches(tok, people=people)[0]
28 with Tap(model) as tap:
29 logits, _ = model(batch.clean)
30 parts = lens.attribute(model, tap, batch.right, batch.wrong, position=4)
31 total = parts["embed"] + parts["heads"].sum((1, 2)) + parts["mlp"].sum(1)
32 assert torch.allclose(total, tasks.logit_diff(logits[:, 4], batch.right, batch.wrong), atol=1e-4)