minterp

tests/test_lens.py

32 lines · View on GitHub · Linked from 1. Read

1import torch
2
3from minterp import lens, tasks
4from minterp.tap import Tap
5
6
7def test_decoding_the_final_stream_gives_the_logits(model, tok, people):
8    batch = tasks.batches(tok, people=people)[0]
9    with Tap(model) as tap:
10        logits, _ = model(batch.clean)
11    assert torch.allclose(lens.decode(model, tap["resid", model.cfg.n_layers]), logits, atol=1e-5)
12
13
14def test_the_attribution_adds_up_to_the_logit_difference(model, tok, people):
15    for batch in tasks.batches(tok, people=people):
16        with Tap(model) as tap:
17            logits, _ = model(batch.clean)
18        parts = lens.attribute(model, tap, batch.right, batch.wrong)
19        n, cfg = len(batch), model.cfg
20        assert parts["embed"].shape == (n,)
21        assert parts["heads"].shape == (n, cfg.n_layers, cfg.n_heads) and parts["mlp"].shape == (n, cfg.n_layers)
22        total = parts["embed"] + parts["heads"].sum((1, 2)) + parts["mlp"].sum(1)
23        assert torch.allclose(total, tasks.logit_diff(logits, batch.right, batch.wrong), atol=1e-4)
24
25
26def test_the_attribution_of_a_position_other_than_the_last(model, tok, people):
27    batch = tasks.batches(tok, people=people)[0]
28    with Tap(model) as tap:
29        logits, _ = model(batch.clean)
30    parts = lens.attribute(model, tap, batch.right, batch.wrong, position=4)
31    total = parts["embed"] + parts["heads"].sum((1, 2)) + parts["mlp"].sum(1)
32    assert torch.allclose(total, tasks.logit_diff(logits[:, 4], batch.right, batch.wrong), atol=1e-4)