class HCPrenormGemmKernel(VllmJitKernel["HCPrenormGemmKernel.CompileKey"]):
"""DeepGEMM's `tf32_hc_prenorm_gemm`, which JITs once per (n, k, splits)."""
@dataclass(frozen=True, slots=True)
class CompileKey:
n: int
k: int
n_splits: int
def get_warmup_keys(
self,
*,
max_tokens: int,
n: int,
k: int,
num_splits: Callable[[int, int], int],
) -> list[CompileKey]:
# num_splits is the caller's runtime heuristic; it moves every 64 tokens.
splits = {
num_splits(k, num_tokens) for num_tokens in range(1, max_tokens + 1, 64)
}
return [self.CompileKey(n=n, k=k, n_splits=s) for s in sorted(splits)]
def compile(self, compile_key: CompileKey) -> None:
if compile_key in self._compiled_cache:
return
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
n, k, splits = compile_key.n, compile_key.k, compile_key.n_splits
# M is a launch argument, not a template parameter, so any M works.
num_tokens = 64
device = current_platform.device_type
x = torch.zeros((num_tokens, k), device=device, dtype=torch.bfloat16)
fn = torch.zeros((n, k), device=device, dtype=torch.float32)
out = torch.empty((splits, num_tokens, n), device=device, dtype=torch.float32)
sqr_sum = torch.empty((splits, num_tokens), device=device, dtype=torch.float32)
tf32_hc_prenorm_gemm(x, fn, out, sqr_sum, splits)
self._compiled_cache[compile_key] = True