minterp

minterp/patch.py

80 lines · View on GitHub · Linked from 3. Patch

1"""
2Patching: run a prompt with a piece of another prompt's run in its place.
3
4Reading (step 1) and probing (step 2) look at a run and leave it alone. A
5patch changes it. Record the activations of a source prompt, run a target
6prompt, and at one place put the source's activation where the target's own
7would be. Everything downstream computes on. If the answer moves towards
8the source's answer, the model was carrying, at that place, something the
9answer depends on.
10
11    source, _ = record(model, batch.clean)
12    at = [batch.pos["S"]]  # the acting name
13    logits = run(model, batch.flipped, {("resid", 6): swap(source["resid", 6], positions=at)})
14
15The two prompts must line up token by token, which is why the versions of
16minterp/tasks.py differ in a single token.
17
18Three things can stand in for an activation, and they ask different
19questions (docs/03-patch.md):
20
21    another prompt's   swap(source[key])        what if this place had seen the other prompt?
22    the average        swap(mean(stores, key))  what if this place carried nothing about this prompt?
23    nothing            swap(zero(source[key]))  what if this place wrote nothing at all?
24"""
25
26import torch
27
28from .tap import KINDS, Tap
29
30
31@torch.no_grad()
32def record(model, tokens, keep=KINDS):
33    """One forward pass, with everything a patch may need. Returns the store of the Tap and the logits."""
34    with Tap(model, keep=keep) as tap:
35        logits, _ = model(tokens)
36    return tap.store, logits
37
38
39@torch.no_grad()
40def run(model, tokens, edits):
41    """The logits of a forward pass with some activations replaced. `edits` maps a Tap key to a function."""
42    with Tap(model, edit=edits, keep=()):
43        logits, _ = model(tokens)
44    return logits
45
46
47def swap(source, positions=None, head=None):
48    """
49    An edit that puts `source` in place of the activation: everywhere, at
50    some positions only, or for one head only (for a ("heads", block) key,
51    whose activation is (batch, seq, heads, head_dim)).
52    """
53    def edit(x):
54        x = x.clone()
55        at = slice(None) if positions is None else positions
56        if head is None:
57            x[:, at] = source[:, at]
58        else:
59            x[:, at, head] = source[:, at, head]
60        return x
61    return edit
62
63
64def mean(stores, key):
65    """The average of an activation over the prompts of several runs, position by position."""
66    return torch.cat([s[key] for s in stores]).mean(0, keepdim=True)
67
68
69def zero(activation):
70    return torch.zeros_like(activation)
71
72
73def share(patched, target, source):
74    """
75    How far a patch moved a measure from the target run's value towards the
76    source run's: 0 is no movement, 1 is all the way. The three arguments
77    are the measure per prompt; the share is taken on their means, because a
78    ratio per prompt blows up wherever the two runs happen to agree.
79    """
80    return ((patched.mean() - target.mean()) / (source.mean() - target.mean())).item()