From bf09a35c959dd76e5e284b90959efc5b3000ecb4 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Sat, 27 Jun 2026 16:30:34 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20optimizer=20=E5=8F=82=E6=95=B0=E5=88=86?=
 =?UTF-8?q?=E7=BB=84=EF=BC=8Cbias/norm=20=E4=B8=8D=E5=81=9A=20weight=20dec?=
 =?UTF-8?q?ay?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 scripts/tools/train.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/scripts/tools/train.py b/scripts/tools/train.py
index 2f3752c..5c8611a 100644
--- a/scripts/tools/train.py
+++ b/scripts/tools/train.py
@@ -269,7 +269,20 @@ def create_model(config):
 
 
 def create_optimizer(model, **kwargs) -> optim.Optimizer:
-    return optim.AdamW(model.parameters(), fused=True, **kwargs)
+    decay_params = []
+    no_decay_params = []
+    for name, param in model.named_parameters():
+        if not param.requires_grad:
+            continue
+        if param.dim() < 2 or "norm" in name or "bias" in name:
+            no_decay_params.append(param)
+        else:
+            decay_params.append(param)
+    param_groups = [
+        {"params": decay_params, "weight_decay": kwargs.pop("weight_decay", 0.01)},
+        {"params": no_decay_params, "weight_decay": 0.0},
+    ]
+    return optim.AdamW(param_groups, fused=True, **kwargs)
 
 
 def create_scheduler(