With no starter code, implement a GPT-style transformer decoder from scratch in about 30 minutes.

import torch, torch.nn as nn, torch.nn.functional as F

class Block(nn.Module):
    def __init__(self, d, h):
        super().__init__()
        self.h = h
        self.ln1 = nn.LayerNorm(d)
        self.ln2 = nn.LayerNorm(d)
        self.qkv = nn.Linear(d, 3 * d)
        self.proj = nn.Linear(d, d)
        self.mlp = nn.Sequential(
            nn.Linear(d, 4 * d), nn.GELU(),
            nn.Linear(4 * d, d))

    def attn(self, x):
        B, T, C = x.shape
        q, k, v = self.qkv(x).split(C, dim=-1)
        q, k, v = (t.view(B, T, self.h, -1)
                   .transpose(1, 2) for t in (q, k, v))
        s = q @ k.transpose(-2, -1) / k.shape[-1] ** 0.5
        m = torch.ones(T, T, dtype=torch.bool,
                       device=x.device).tril()
        s = s.masked_fill(~m, float("-inf"))
        y = s.softmax(-1) @ v
        y = y.transpose(1, 2).reshape(B, T, C)
        return self.proj(y)

    def forward(self, x):
        x = x + self.attn(self.ln1(x))
        return x + self.mlp(self.ln2(x))

class Decoder(nn.Module):
    def __init__(self, vocab, d, h, n, max_len):
        super().__init__()
        self.tok = nn.Embedding(vocab, d)
        self.pos = nn.Embedding(max_len, d)
        self.blocks = nn.ModuleList(
            Block(d, h) for _ in range(n))
        self.ln = nn.LayerNorm(d)
        self.head = nn.Linear(d, vocab, bias=False)

    def forward(self, idx, targets=None):
        pos = torch.arange(idx.shape[1],
                           device=idx.device)
        x = self.tok(idx) + self.pos(pos)
        for b in self.blocks:
            x = b(x)
        logits = self.head(self.ln(x))
        if targets is None:
            return logits
        loss = F.cross_entropy(logits.flatten(0, 1),
                               targets.flatten())
        return logits, loss

Interviewers watch the causal mask, head reshapes that keep batch and heads apart, pre-norm residuals, and targets shifted by one token outside the model.