From f433672140dd317b9b2819bbc57780d4d8abf42a Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Sat, 18 Jul 2026 23:48:08 +0800
Subject: [PATCH] fix: use sum reduction for DPO sequence logprob

- DPO requires sequence-level sum of token logprobs, not per-token mean
- mean reduction made beta*ratio_diff ~0.03 (near-zero gradient)
- loss stalled at 0.6931 because logsigmoid(0.03) has vanishing grad
- sum gives beta*ratio_diff ~10 with meaningful gradients
---
 astrai/trainer/strategy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/astrai/trainer/strategy.py b/astrai/trainer/strategy.py
index 55c5468..e4acc89 100644
--- a/astrai/trainer/strategy.py
+++ b/astrai/trainer/strategy.py
@@ -225,7 +225,7 @@ class DPOStrategy(BaseStrategy):
         device: str,
         ref_model: nn.Module,
         beta: float = 0.1,
-        reduction: str = "mean",
+        reduction: str = "sum",
         **kwargs,
     ):
         super().__init__(model, device, **kwargs)