refactor(dataset): 重构数据集处理流程并添加新功能

2025-07-16 09:56:31 +08:00
parent 6edc31ce2e
commit b69ba2c5ff
8 changed files with 30 additions and 13 deletions
@@ -0,0 +1,10 @@
+from datasets import load_dataset
+from utils import pre_tarin_process
+
+if __name__ == "__main__":
+    dataset = load_dataset("BelleGroup/train_3.5M_CN")
+    # pre_tarin_process(
+    #     dataset_dict=dataset,
+    #     output_subdir="belle_sft",
+    #     max_chunk_size=5,
+    # )