fix: let flash backend handle 4D causal prefill mask
- Treat 4D masks as causal (flash handles it natively), keep rejecting custom non-causal masks - Enables flash backend in benchmark --compare and real prefill path
This commit is contained in:
@@ -140,7 +140,9 @@ def _backend_supports(
|
|||||||
return False
|
return False
|
||||||
if q.size(1) == 1 and kv_cache is not None:
|
if q.size(1) == 1 and kv_cache is not None:
|
||||||
return True
|
return True
|
||||||
return attn_mask is None
|
if attn_mask is None or is_causal:
|
||||||
|
return True
|
||||||
|
return attn_mask.dim() == 4
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
@@ -640,7 +642,7 @@ class FlashAttnBackend(AttentionBackend):
|
|||||||
k = repeat_kv(k, n_rep)
|
k = repeat_kv(k, n_rep)
|
||||||
v = repeat_kv(v, n_rep)
|
v = repeat_kv(v, n_rep)
|
||||||
|
|
||||||
if attn_mask is not None and not is_causal:
|
if attn_mask is not None and not is_causal and attn_mask.dim() != 4:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"FlashAttnBackend does not support a custom attention mask; "
|
"FlashAttnBackend does not support a custom attention mask; "
|
||||||
"use a causal mask or select TorchNativeBackend."
|
"use a causal mask or select TorchNativeBackend."
|
||||||
@@ -652,7 +654,10 @@ class FlashAttnBackend(AttentionBackend):
|
|||||||
"Install with `pip install flash-attn`."
|
"Install with `pip install flash-attn`."
|
||||||
)
|
)
|
||||||
out = fa.flash_attn_func(
|
out = fa.flash_attn_func(
|
||||||
q.contiguous(), k.contiguous(), v.contiguous(), causal=is_causal
|
q.contiguous(),
|
||||||
|
k.contiguous(),
|
||||||
|
v.contiguous(),
|
||||||
|
causal=is_causal or (attn_mask is not None and attn_mask.dim() == 4),
|
||||||
)
|
)
|
||||||
return out.contiguous().flatten(2)
|
return out.contiguous().flatten(2)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user