From 8725a9ebd147465128899162bfbac8e4c321add0 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Fri, 1 Aug 2025 16:59:47 +0800
Subject: [PATCH] =?UTF-8?q?feat(dump=5Fsft=5Ffile):=20=E6=9B=B4=E6=96=B0?=
 =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=9B=86=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 dump_sft_file.py |  4 ++--
 sft_belle.py     | 19 +++++--------------
 2 files changed, 7 insertions(+), 16 deletions(-)

diff --git a/dump_sft_file.py b/dump_sft_file.py
index 1c7a9ca..6b11738 100644
--- a/dump_sft_file.py
+++ b/dump_sft_file.py
@@ -14,7 +14,7 @@ def get_processor(tokenizer: BpeTokenizer):
         tokens = prefix_ids + suffix_ids
         tokens = torch.tensor(tokens, dtype=torch.int32)
         masks = torch.zeros_like(tokens, dtype=torch.bool)
-        masks[:len(prefix_ids)] = True
+        masks[len(prefix_ids):] = True
         
         return {"sequence": tokens, "mask": masks}
     
@@ -24,7 +24,7 @@ def get_processor(tokenizer: BpeTokenizer):
 if __name__ == "__main__":
     tokenizer = BpeTokenizer("tokenizer.json")
     base_dir = [
-        # os.path.join("dataset", "belle-sft"),
+        os.path.join("dataset", "Ling-Coder-SFT"),
         os.path.join("dataset", "chinese-instruct")
     ]
     base_out_dir = "pkl_output"
diff --git a/sft_belle.py b/sft_belle.py
index 50aab61..f86b04d 100644
--- a/sft_belle.py
+++ b/sft_belle.py
@@ -2,15 +2,6 @@ from datasets import load_dataset
 from utils import process_dataset
 
 
-def build_prompt(query, history) -> str:
-    ret_prompt = ""
-    if len(history) > 0:
-        for his_query, his_response in history:
-            ret_prompt += f"<|user|> {his_query} <|system|> <bos>{his_response}<eos>\n"
-    if query is not None:
-        ret_prompt += f"<|user|> {query} <|system|> <bos>"
-    return ret_prompt
-
 
 def process_func(input_dict: dict):
     conversations = input_dict["conversations"]
@@ -20,12 +11,12 @@ def process_func(input_dict: dict):
     for i in range(n):
         user_msg = conversations[2*i]["value"]
         assistant_msg = conversations[2*i+1]["value"]
-        examples.append((user_msg, assistant_msg))
+        examples.append({
+            "query": user_msg,
+            "response": assistant_msg
+        })
     
-    content = {
-        "text": build_prompt(None, examples)
-    }
-    return content
+    return examples
 
 
 if __name__ == "__main__":