Implement a client that spreads LLM calls across several providers, respecting their rate limits and failing over when one is down.

Try providers in preference order, and bench one for its retry-after time when it rate-limits or errors.

import random, time

class RateLimited(Exception):
    retry_after = None

class Pool:
    def __init__(self, providers, cooldown=30.0):
        self.ps = providers  # in preference order
        self.until = {p.name: 0.0 for p in providers}
        self.cooldown = cooldown

    def call(self, req, rounds=3):
        for r in range(rounds):
            for p in self.ps:
                now = time.monotonic()
                if self.until[p.name] > now:
                    continue
                try:
                    return p.complete(req)
                except RateLimited as e:
                    wait = e.retry_after or self.cooldown
                    self.until[p.name] = now + wait
                except (TimeoutError, ConnectionError):
                    self.until[p.name] = now + self.cooldown
            time.sleep(2 ** r + random.random())
        raise RuntimeError("all providers failed")

Interviewers look for jittered backoff, never retrying a bad request, timeouts on every call, and no silent failover mid-stream once tokens reach the user.