tests/test_tasks.py
57 lines · View on GitHub · Linked from 1. Read
1import torch
2
3from minterp import tasks
4from minterp.common import eot
5
6
7def test_names_keeps_single_tokens(tok, people):
8 assert tasks.names(tok, people) == people
9
10 class Splits(type(tok)):
11 def encode_ordinary(self, text):
12 return [1, 2] if text == " " + people[0] else super().encode_ordinary(text)
13
14 two = Splits.__new__(Splits)
15 two.__dict__ = tok.__dict__
16 assert tasks.names(two, people) == people[1:]
17
18
19def test_the_three_versions_differ_in_one_token(tok, people):
20 for batch in tasks.batches(tok, people=people):
21 n = len(people) * (len(people) - 1)
22 assert len(batch) == n and batch.clean.shape == batch.flipped.shape == batch.neutral.shape
23 assert (batch.clean[:, 0] == eot(tok)).all()
24 s = batch.pos["S"]
25 for other in (batch.flipped, batch.neutral):
26 differs = other != batch.clean
27 assert differs[:, s].all() and differs.sum() == n
28 assert batch.pos["A"] < batch.pos["B"] < s < batch.pos["end"] == batch.clean.shape[1] - 1
29 assert (batch.clean[:, batch.pos["A"]] == batch.wrong).all()
30 assert (batch.clean[:, batch.pos["B"]] == batch.right).all()
31 assert (batch.clean[:, s] == batch.wrong).all()
32 assert (batch.flipped[:, s] == batch.right).all()
33 for (a, b, c), row in zip(batch.names, batch.neutral):
34 assert len({a, b, c}) == 3 and tok.decode([row[s].item()]) == " " + c
35
36 offsets = {(people.index(c) - people.index(b)) % len(people) for _, b, c in batch.names}
37 assert offsets == set(range(1, len(people)))
38 counts = [sum(c == n for _, _, c in batch.names) for n in people]
39 assert max(counts) - min(counts) <= 2
40
41 differs = batch.other != batch.clean
42 assert differs[:, batch.pos["B"]].all() and differs.sum() == n
43 assert (batch.other[:, batch.pos["B"]] == batch.third).all()
44 differs = batch.other_first != batch.flipped
45 assert differs[:, batch.pos["A"]].all() and differs.sum() == n
46 assert (batch.other_first[:, batch.pos["A"]] == batch.third).all()
47 assert [batch.labels[batch.pos[k]] for k in ("A", "B", "S")] == ["A", "B", "S"]
48 assert len(batch.labels) == batch.clean.shape[1]
49
50
51def test_logit_diff_reads_the_last_position():
52 logits = torch.zeros(2, 3, 5)
53 logits[0, -1, 4], logits[0, -1, 1] = 3.0, 1.0
54 logits[1, 0, 4] = 9.0
55 right, wrong = torch.tensor([4, 4]), torch.tensor([1, 1])
56 assert tasks.logit_diff(logits, right, wrong).tolist() == [2.0, 0.0]
57 assert tasks.logit_diff(logits[:, -1], right, wrong).tolist() == [2.0, 0.0]