refactor: 重构数据管道模块

supervised_finetuning/sft_belle.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from modules.utils import process_dataset
+from modules.datapipeline import DataPipeline
 
 
 
@@ -21,7 +21,9 @@ def process_func(input_dict: dict):
 
 if __name__ == "__main__":
     dataset = load_dataset("BelleGroup/train_3.5M_CN")
-    process_dataset(
+    
+    pipeline = DataPipeline()
+    pipeline.process_dataset(
         dataset_dict=dataset,
         output_subdir="belle-sft",
         process_func=process_func,

supervised_finetuning/sft_chinese_instruct.py

@@ -1,14 +1,15 @@
 from datasets import DatasetDict
 from datasets import load_dataset, concatenate_datasets
-from modules.utils import process_dataset, comprehensive_normalization
+from modules.datapipeline import DataPipeline, TextNormalizer
 
 
 def process_func(input_dict: dict):
     query = input_dict["prompt"] if input_dict["prompt"] else ""
     resp = input_dict["response"] if input_dict["response"] else ""
     
-    query = comprehensive_normalization(query)
-    resp = comprehensive_normalization(resp)
+    normalizer = TextNormalizer()
+    query = normalizer.normalize(query)
+    resp = normalizer.normalize(resp)
     
     return {"query": query, "response": resp }
 
@@ -25,7 +26,8 @@ if __name__ == "__main__":
     
     combined_dataset = concatenate_datasets(datasets)
     
-    process_dataset(
+    pipeline = DataPipeline()
+    pipeline.process_dataset(
         dataset_dict=DatasetDict({"train": combined_dataset}),
         output_subdir="chinese-instruct-sft",
         process_func=process_func,

supervised_finetuning/sft_coder.py

@@ -1,6 +1,6 @@
 # inclusionAI/Ling-Coder-SFT
 from datasets import load_dataset
-from modules.utils import process_dataset
+from modules.datapipeline import DataPipeline
 
 
 def process_func(input_dict: dict) -> dict:
@@ -13,7 +13,8 @@ def process_func(input_dict: dict) -> dict:
 if __name__ == "__main__":
     dataset = load_dataset("inclusionAI/Ling-Coder-SFT")
     
-    process_dataset(
+    pipeline = DataPipeline()
+    pipeline.process_dataset(
         dataset_dict=dataset,
         output_subdir="Ling-Coder-sft",
         process_func=process_func

supervised_finetuning/sft_firefly-1.1m-rephrased.py

@@ -1,6 +1,6 @@
 # Mxode/Firefly-1.1M-Rephrased
 from datasets import load_dataset
-from modules.utils import process_dataset
+from modules.datapipeline import DataPipeline
 
 def process_func(input_dict: dict):
     instruction = input_dict["instruction"]
@@ -11,7 +11,8 @@ def process_func(input_dict: dict):
 if __name__ == "__main__":
     dataset = load_dataset("Mxode/Firefly-1.1M-Rephrased")
     
-    process_dataset(
+    pipeline = DataPipeline()
+    pipeline.process_dataset(
         dataset_dict=dataset,
         output_subdir="Firefly-1.1M-Rephrased",
         process_func=process_func

supervised_finetuning/sft_magpie-pro-300k.py

@@ -1,6 +1,6 @@
 # HuggingFaceTB/Magpie-Pro-300K-Filtered-H4
 from datasets import load_dataset
-from modules.utils import process_dataset
+from modules.datapipeline import DataPipeline
 
 
 def process_func(input_dict: dict):
@@ -22,7 +22,9 @@ def process_func(input_dict: dict):
 
 if __name__ == "__main__":
     dataset = load_dataset("HuggingFaceTB/Magpie-Pro-300K-Filtered-H4")
-    process_dataset(
+    
+    pipeline = DataPipeline()
+    pipeline.process_dataset(
         dataset_dict=dataset,
         output_subdir="Magpie-Pro-300K-sft",
         process_func=process_func,