"""One attention query with an optional causal mask."""

from math import exp, isclose, sqrt


def attention(query, keys, values, position):
    scores = [
        sum(left * right for left, right in zip(query, key, strict=True)) / sqrt(len(query))
        for key in keys[: position + 1]
    ]
    # Shift before exponentiating to improve numerical stability.
    maximum = max(scores)
    masses = [exp(score - maximum) for score in scores]
    total = sum(masses)
    weights = [mass / total for mass in masses]
    weights.extend([0.0] * (len(keys) - len(weights)))
    output = [
        sum(weight * value for weight, value in zip(weights, column, strict=True))
        for column in zip(*values, strict=True)
    ]
    return weights, output


if __name__ == "__main__":
    keys = [(1, 0), (0, 1), (10, 0)]
    values = [(2, 0), (0, 2), (30, 30)]
    weights, output = attention((1, 0), keys, values, position=1)
    assert isclose(sum(weights), 1)
    assert weights[2] == 0.0
    assert isclose(output[0], 1.33952, rel_tol=1e-5)
    print("weights =", [round(value, 3) for value in weights])
    print("output  =", [round(value, 3) for value in output])
