tests/test_probe.py
77 lines · View on GitHub · Linked from 2. Probe
1import torch
2
3from minterp import probe
4
5
6def toy(n=240, d=20, k=4, places=3, noise=0.3, seed=0):
7 """Rows whose label is written along k fixed directions, at every place."""
8 g = torch.Generator().manual_seed(seed)
9 labels = torch.randint(0, k, (n,), generator=g)
10 directions = torch.randn(places, k, d, generator=g, dtype=torch.float64)
11 X = directions[:, labels] + noise * torch.randn(places, n, d, generator=g, dtype=torch.float64)
12 return X, labels
13
14
15def test_fit_solves_the_ridge_problem():
16 X, labels = toy()
17 (X,) = probe.standardize(X)
18 Y = probe.onehot(labels[None], 4)
19 W, b = probe.fit(X, Y, 0.1)
20
21 n = X.shape[1]
22 gradient = X.transpose(-1, -2) @ (X @ W + b - Y) + n * 0.1 * W
23 assert gradient.abs().max() < 1e-8
24 assert torch.allclose(b, Y.mean(0))
25
26 assert probe.fit(X, Y, 10.0)[0].norm() < W.norm()
27
28
29def test_a_written_label_is_read_and_a_random_one_is_not():
30 X, labels = toy()
31 g = torch.Generator().manual_seed(1)
32 random = torch.randint(0, 4, labels.shape, generator=g)
33 groups = torch.arange(len(labels)) % 4
34 train, test = slice(0, 180), slice(180, 240)
35 out, chosen = probe.read(X[:, train], torch.stack([labels, random])[:, train], groups[train], X[:, test], 4)
36 assert out.shape == (3, 60, 2, 4) and chosen.shape == (3, 2)
37 guess = out.argmax(-1)
38 assert (guess[:, :, 0] == labels[test]).double().mean() > 0.95
39 assert (guess[:, :, 1] == random[test]).double().mean() < 0.45
40
41
42def test_two_runs_give_the_same_numbers_and_scale_does_not_matter():
43 X, labels = toy()
44 groups = torch.arange(len(labels)) % 4
45 args = (labels[None, :180], groups[:180])
46 one, _ = probe.read(X[:, :180], *args, X[:, 180:], 4)
47 two, _ = probe.read(X[:, :180], *args, X[:, 180:], 4)
48 assert torch.equal(one, two)
49
50 big, _ = probe.read(60 * X[:, :180], *args, 60 * X[:, 180:], 4)
51 assert torch.allclose(one, big, atol=1e-9)
52
53
54def test_standardize_uses_the_training_rows_only():
55 X, _ = toy()
56 train, test = probe.standardize(X[:, :100], X[:, 100:])
57 assert train.mean(1).abs().max() < 1e-12 and torch.allclose(train.std(1), torch.ones(3, 20, dtype=torch.float64))
58 shifted = probe.standardize(X[:, :100], X[:, 100:] + 5.0)[1]
59 assert torch.allclose(shifted - test, 5.0 / X[:, :100].std(1, keepdim=True).expand_as(test))
60
61
62def test_onehot_puts_the_targets_side_by_side():
63 y = probe.onehot(torch.tensor([[0, 2], [1, 1]]), 3)
64 assert y.tolist() == [[1, 0, 0, 0, 1, 0], [0, 0, 1, 0, 1, 0]]
65
66
67def test_weights_are_the_reader_that_read_applies():
68 X, labels = toy()
69 groups = torch.arange(len(labels)) % 4
70 out, _ = probe.read(X[:1, :180], labels[None, :180], groups[:180], X[:1, 180:], 4)
71 W, b, mean, spread = probe.weights(X[0, :180], labels[:180], groups[:180], 4)
72 again = ((X[0, 180:] - mean) / spread) @ W + b
73 assert torch.allclose(again, out[0, :, 0], atol=1e-9)
74
75 part = 0.3 * X[0, 180:]
76 rest = X[0, 180:] - part
77 assert torch.allclose(((part - mean) / spread) @ W + b + (rest / spread) @ W, again, atol=1e-9)