refactor: namespace csrc kernels and extract common helpers
- attention family -> astrai::attention; fp8 family -> astrai::fp8 - new common/reduce.cuh (warp/group reductions, atomic_max_float) - new common/cp_async.cuh (predicated cp_async_16, commit/wait group) - move MAX_SPLITS into attention/common.h; delete warp_utils.cuh - .cu bindings and pure C tests open family namespaces via using
This commit is contained in:
@@ -0,0 +1,69 @@
|
||||
// Shared cp.async primitives — pure CUDA, no torch.
|
||||
//
|
||||
// One header for the async-copy pipeline used by both the attention kernels
|
||||
// (predicated 16-byte K/V tile staging) and the fp8 GEMM (predicated operand
|
||||
// staging + wait_group dispatch). PTX requires wait_group's operand to be an
|
||||
// immediate, hence the template forms.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
namespace astrai {
|
||||
|
||||
// Predicated cp.async: copy 16 bytes when `pred`, otherwise zero-fill.
|
||||
// src_size=0 means no bytes are read, so an out-of-bounds address is safe.
|
||||
// BypassL1 defaults to .cg (L2 only); false selects .ca (L1 + L2).
|
||||
// `T` is the smem element type; only the destination pointer's type matters.
|
||||
template <typename T, bool BypassL1 = true>
|
||||
__device__ __forceinline__ void cp_async_16(T* smem_ptr, const void* gmem_ptr,
|
||||
bool pred) {
|
||||
const unsigned smem_addr = __cvta_generic_to_shared(smem_ptr);
|
||||
const int src_size = pred ? 16 : 0;
|
||||
if constexpr (BypassL1) {
|
||||
asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;"
|
||||
:: "r"(smem_addr), "l"(gmem_ptr), "r"(src_size));
|
||||
} else {
|
||||
asm volatile("cp.async.ca.shared.global [%0], [%1], 16, %2;"
|
||||
:: "r"(smem_addr), "l"(gmem_ptr), "r"(src_size));
|
||||
}
|
||||
}
|
||||
|
||||
// Commit all outstanding cp.async ops of this thread as one group.
|
||||
__device__ __forceinline__ void cp_async_commit_group() {
|
||||
asm volatile("cp.async.commit_group;");
|
||||
}
|
||||
|
||||
// Wait for every committed group (pipeline drain).
|
||||
__device__ __forceinline__ void cp_async_wait_all() {
|
||||
asm volatile("cp.async.wait_all;");
|
||||
}
|
||||
|
||||
// Wait until at most KeepGroups committed groups are still in flight.
|
||||
// PTX requires an immediate operand; keep it as a template argument so the
|
||||
// stage policy stays compile-time configurable.
|
||||
template <int KeepGroups>
|
||||
__device__ __forceinline__ void cp_async_wait_group() {
|
||||
static_assert(KeepGroups >= 0 && KeepGroups <= 7,
|
||||
"cp.async.wait_group supports immediates in [0, 7]");
|
||||
asm volatile("cp.async.wait_group %0;" :: "n"(KeepGroups));
|
||||
}
|
||||
|
||||
// Runtime dispatch over cp_async_wait_group<N>: unrolls into a compare
|
||||
// ladder over [0, MaxKeepGroups] so the immediate-only PTX constraint is
|
||||
// hidden behind a runtime `keep_groups` (used by the fp8 GEMM pipeline,
|
||||
// whose remaining-tile count is dynamic).
|
||||
template <int MaxKeepGroups>
|
||||
__device__ __forceinline__ void cp_async_wait_group_dispatch(int keep_groups) {
|
||||
static_assert(MaxKeepGroups >= 0 && MaxKeepGroups <= 7,
|
||||
"cp.async.wait_group supports immediates in [0, 7]");
|
||||
if (keep_groups == MaxKeepGroups) {
|
||||
cp_async_wait_group<MaxKeepGroups>();
|
||||
} else if constexpr (MaxKeepGroups > 0) {
|
||||
cp_async_wait_group_dispatch<MaxKeepGroups - 1>(keep_groups);
|
||||
} else {
|
||||
cp_async_wait_group<0>();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace astrai
|
||||
@@ -0,0 +1,48 @@
|
||||
// Shared warp/block reduction + atomic helpers — pure CUDA, no torch.
|
||||
//
|
||||
// Extracted from the attention and fp8 families so both share one
|
||||
// implementation: warp_reduce_sum (decode scalar kernel), warp_reduce_max +
|
||||
// atomic_max_float (fp8 quantize amax), group_reduce_sum<G> (prefill scalar
|
||||
// kernel).
|
||||
|
||||
#pragma once
|
||||
|
||||
namespace astrai {
|
||||
|
||||
// Full-warp butterfly sum reduction (32 lanes).
|
||||
__device__ __forceinline__ float warp_reduce_sum(float val) {
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset >>= 1)
|
||||
val += __shfl_xor_sync(0xFFFFFFFF, val, offset);
|
||||
return val;
|
||||
}
|
||||
|
||||
// Full-warp butterfly max reduction (32 lanes).
|
||||
__device__ __forceinline__ float warp_reduce_max(float value) {
|
||||
#pragma unroll
|
||||
for (int offset = 16; offset > 0; offset >>= 1)
|
||||
value = fmaxf(value, __shfl_xor_sync(0xffffffffu, value, offset));
|
||||
return value;
|
||||
}
|
||||
|
||||
// Sub-warp group reduction over G consecutive lanes (G a power of two).
|
||||
// `mask` is the full participating-lane mask of the group (see the
|
||||
// prefill scalar kernel's gmask computation).
|
||||
template <int G>
|
||||
__device__ __forceinline__ float group_reduce_sum(float v, unsigned mask) {
|
||||
#pragma unroll
|
||||
for (int o = G / 2; o > 0; o >>= 1)
|
||||
v += __shfl_xor_sync(mask, v, o);
|
||||
return v;
|
||||
}
|
||||
|
||||
// Unsigned-bit-pattern atomicMax for non-negative floats; a null
|
||||
// destination disables the update (kernels with optional amax slots).
|
||||
__device__ __forceinline__ void atomic_max_float(float* destination,
|
||||
float value) {
|
||||
if (destination)
|
||||
atomicMax(reinterpret_cast<unsigned*>(destination),
|
||||
__float_as_uint(value));
|
||||
}
|
||||
|
||||
} // namespace astrai
|
||||
Reference in New Issue
Block a user