feat(dump_sft_file): 更新数据集处理逻辑

sft_belle.py

@@ -2,15 +2,6 @@ from datasets import load_dataset
 from utils import process_dataset
 
 
-def build_prompt(query, history) -> str:
-    ret_prompt = ""
-    if len(history) > 0:
-        for his_query, his_response in history:
-            ret_prompt += f"<|user|> {his_query} <|system|> <bos>{his_response}<eos>\n"
-    if query is not None:
-        ret_prompt += f"<|user|> {query} <|system|> <bos>"
-    return ret_prompt
-
 
 def process_func(input_dict: dict):
     conversations = input_dict["conversations"]
@@ -20,12 +11,12 @@ def process_func(input_dict: dict):
     for i in range(n):
         user_msg = conversations[2*i]["value"]
         assistant_msg = conversations[2*i+1]["value"]
-        examples.append((user_msg, assistant_msg))
+        examples.append({
+            "query": user_msg,
+            "response": assistant_msg
+        })
     
-    content = {
-        "text": build_prompt(None, examples)
-    }
-    return content
+    return examples
 
 
 if __name__ == "__main__":