From 44be73d2550a2f6b1fa967816617b3545e279197 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Fri, 12 Sep 2025 13:46:04 +0800
Subject: [PATCH] =?UTF-8?q?refactor(supervised=5Ffinetuning):=20=E9=87=8D?=
 =?UTF-8?q?=E6=9E=84=20Chinese=20Instruct=20=E6=95=B0=E6=8D=AE=E9=9B=86?=
 =?UTF-8?q?=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 supervised_finetuning/sft_chinese_instruct.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/supervised_finetuning/sft_chinese_instruct.py b/supervised_finetuning/sft_chinese_instruct.py
index 7317332..ccbef6c 100644
--- a/supervised_finetuning/sft_chinese_instruct.py
+++ b/supervised_finetuning/sft_chinese_instruct.py
@@ -1,23 +1,14 @@
 from datasets import DatasetDict
 from datasets import load_dataset, concatenate_datasets
-from modules.utils import process_dataset
+from modules.utils import process_dataset, comprehensive_normalization
 
 
-def replace_seg(query:str, response:str) -> str:
-    replacements = {
-        "\\[": "$$", "\\]": "$$",
-        "\\(": "$", "\\)": "$"
-    }
-    for old, new in replacements.items():
-        query = query.replace(old, new)
-        response = response.replace(old, new)
-
-    return query, response
-
 def process_func(input_dict: dict):
     query = input_dict["prompt"] if input_dict["prompt"] else ""
     resp = input_dict["response"] if input_dict["response"] else ""
-    query, resp = replace_seg(query, resp)
+    
+    query = comprehensive_normalization(query)
+    resp = comprehensive_normalization(resp)
     
     return {"query": query, "response": resp }