feat(project_structure): 重构项目目录并添加运行脚本

supervised_finetuning/__init__.py

@@ -0,0 +1,12 @@
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import utils
+from tokenizer import BpeTokenizer
+
+__all__ = [
+    "utils",
+    "BpeTokenizer",
+]

supervised_finetuning/sft_belle.py

@@ -0,0 +1,28 @@
+from datasets import load_dataset
+from utils import process_dataset
+
+
+
+def process_func(input_dict: dict):
+    conversations = input_dict["conversations"]
+    n = len(conversations) // 2
+    examples = []
+    
+    for i in range(n):
+        user_msg = conversations[2*i]["value"]
+        assistant_msg = conversations[2*i+1]["value"]
+        examples.append({
+            "query": user_msg,
+            "response": assistant_msg
+        })
+    
+    return examples
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("BelleGroup/train_3.5M_CN")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="belle-sft",
+        process_func=process_func,
+    )

supervised_finetuning/sft_chinese_instruct.py

@@ -0,0 +1,41 @@
+from datasets import DatasetDict
+from datasets import load_dataset, concatenate_datasets
+from utils import process_dataset
+
+
+def replace_seg(query:str, response:str) -> str:
+    replacements = {
+        "\\[": "$$", "\\]": "$$",
+        "\\(": "$", "\\)": "$"
+    }
+    for old, new in replacements.items():
+        query = query.replace(old, new)
+        response = response.replace(old, new)
+
+    return query, response
+
+def process_func(input_dict: dict):
+    query = input_dict["prompt"] if input_dict["prompt"] else ""
+    resp = input_dict["response"] if input_dict["response"] else ""
+    query, resp = replace_seg(query, resp)
+    
+    return {"query": query, "response": resp }
+
+
+if __name__ == "__main__":
+    all_data = ['stem_zh', 'infinity-instruct', 'firefly', 'magpie', 'dpsk-r1-distil', 
+                'coig-cqia', 'disc-law', 'neo_sft_phase2', 'chinese-medical', 'chinese-reasoning-distil', 
+                'psycho-10k-dpsk-r1', 'sof-c-zh', 'industryinstruction', 'Chinese-QA-AFAF']
+    
+    datasets = []
+    for subset in all_data:
+        ds = load_dataset("Mxode/Chinese-Instruct", name=subset)
+        datasets.append(ds["train"]) 
+    
+    combined_dataset = concatenate_datasets(datasets)
+    
+    process_dataset(
+        dataset_dict=DatasetDict({"train": combined_dataset}),
+        output_subdir="chinese-instruct",
+        process_func=process_func,
+    )

supervised_finetuning/sft_coder.py

@@ -0,0 +1,21 @@
+# inclusionAI/Ling-Coder-SFT
+from datasets import load_dataset
+from utils import process_dataset
+
+
+def process_func(input_dict: dict) -> dict:
+    msg = input_dict["messages"]
+    query = msg[0]["content"]
+    history = msg[1]["content"]
+    return {"query": query, "response": history}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("inclusionAI/Ling-Coder-SFT")
+    
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="Ling-Coder-SFT",
+        process_func=process_func
+    )
+    

supervised_finetuning/sft_magpie-pro-300k.py

@@ -0,0 +1,30 @@
+# HuggingFaceTB/Magpie-Pro-300K-Filtered-H4
+from datasets import load_dataset
+from utils import process_dataset
+
+
+def process_func(input_dict: dict):
+    conversations = input_dict["conversations"]
+    assert len(conversations) % 2 == 0
+    n = len(conversations) // 2
+    examples = []
+    
+    for i in range(n):
+        user_msg = conversations[2*i]["value"]
+        assistant_msg = conversations[2*i+1]["value"]
+        examples.append({
+            "query": user_msg,
+            "response": assistant_msg
+        })
+    
+    return examples
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceTB/Magpie-Pro-300K-Filtered-H4")
+    process_dataset(
+        dataset_dict=dataset,
+        output_subdir="belle-sft",
+        process_func=process_func,
+        split_name="train-sft"
+    )