refactor(pretrain): 更新预训练数据集输出目录名称

pre_train/chinese-c4.py

@@ -5,5 +5,5 @@ if __name__ == "__main__":
     dataset = load_dataset("shjwudp/chinese-c4")
     process_dataset(
         dataset_dict=dataset,
-        output_subdir="chinese-c4"
+        output_subdir="chinese-c4-pretrain"
     )

pre_train/chinese-cosmopedia.py

@@ -12,7 +12,7 @@ if __name__ == "__main__":
     
     process_dataset(
         dataset_dict=dataset,
-        output_subdir="chinese-wiki",
+        output_subdir="chinese-wiki-pretrain",
         max_chunk_num=max_chunk_num,
         chunk_size=chunk_size,
     )

pre_train/english-fineweb.py

@@ -5,5 +5,5 @@ if __name__ == "__main__":
     dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
     process_dataset(
         dataset_dict=dataset,
-        output_subdir="english-fineweb",
+        output_subdir="english-fineweb-pretrain",
     )

pre_train/english-wiki.py

@@ -5,6 +5,6 @@ if __name__ == "__main__":
     dataset = load_dataset("Blaze7451/enwiki_structured_content")
     process_dataset(
         dataset_dict=dataset,
-        output_subdir="english-wiki",
+        output_subdir="english-wiki-pretrain",
         max_chunk_size=5,
     )

supervised_finetuning/sft_chinese_instruct.py

@@ -36,6 +36,6 @@ if __name__ == "__main__":
     
     process_dataset(
         dataset_dict=DatasetDict({"train": combined_dataset}),
-        output_subdir="chinese-instruct",
+        output_subdir="chinese-instruct-sft",
         process_func=process_func,
     )

supervised_finetuning/sft_coder.py

@@ -15,7 +15,7 @@ if __name__ == "__main__":
     
     process_dataset(
         dataset_dict=dataset,
-        output_subdir="Ling-Coder-SFT",
+        output_subdir="Ling-Coder-sft",
         process_func=process_func
     )
     

supervised_finetuning/sft_magpie-pro-300k.py

@@ -24,7 +24,7 @@ if __name__ == "__main__":
     dataset = load_dataset("HuggingFaceTB/Magpie-Pro-300K-Filtered-H4")
     process_dataset(
         dataset_dict=dataset,
-        output_subdir="belle-sft",
+        output_subdir="Magpie-Pro-300K-sft",
         process_func=process_func,
-        split_name="train-sft"
+        split_name="train_sft",
     )