diff --git a/pre_train/chinese-c4.py b/pre_train/chinese-c4.py index 1c3f2dc..76449c7 100644 --- a/pre_train/chinese-c4.py +++ b/pre_train/chinese-c4.py @@ -5,5 +5,5 @@ if __name__ == "__main__": dataset = load_dataset("shjwudp/chinese-c4") process_dataset( dataset_dict=dataset, - output_subdir="chinese-c4" + output_subdir="chinese-c4-pretrain" ) \ No newline at end of file diff --git a/pre_train/chinese-cosmopedia.py b/pre_train/chinese-cosmopedia.py index f4a88e5..b6cd249 100644 --- a/pre_train/chinese-cosmopedia.py +++ b/pre_train/chinese-cosmopedia.py @@ -12,7 +12,7 @@ if __name__ == "__main__": process_dataset( dataset_dict=dataset, - output_subdir="chinese-wiki", + output_subdir="chinese-wiki-pretrain", max_chunk_num=max_chunk_num, chunk_size=chunk_size, ) \ No newline at end of file diff --git a/pre_train/english-fineweb.py b/pre_train/english-fineweb.py index ecd5fc3..23e7055 100644 --- a/pre_train/english-fineweb.py +++ b/pre_train/english-fineweb.py @@ -5,5 +5,5 @@ if __name__ == "__main__": dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT") process_dataset( dataset_dict=dataset, - output_subdir="english-fineweb", + output_subdir="english-fineweb-pretrain", ) \ No newline at end of file diff --git a/pre_train/english-wiki.py b/pre_train/english-wiki.py index 2f3f43d..2e7ca2b 100644 --- a/pre_train/english-wiki.py +++ b/pre_train/english-wiki.py @@ -5,6 +5,6 @@ if __name__ == "__main__": dataset = load_dataset("Blaze7451/enwiki_structured_content") process_dataset( dataset_dict=dataset, - output_subdir="english-wiki", + output_subdir="english-wiki-pretrain", max_chunk_size=5, ) \ No newline at end of file diff --git a/supervised_finetuning/sft_chinese_instruct.py b/supervised_finetuning/sft_chinese_instruct.py index 6bda02f..7317332 100644 --- a/supervised_finetuning/sft_chinese_instruct.py +++ b/supervised_finetuning/sft_chinese_instruct.py @@ -36,6 +36,6 @@ if __name__ == "__main__": process_dataset( dataset_dict=DatasetDict({"train": combined_dataset}), - output_subdir="chinese-instruct", + output_subdir="chinese-instruct-sft", process_func=process_func, ) \ No newline at end of file diff --git a/supervised_finetuning/sft_coder.py b/supervised_finetuning/sft_coder.py index c4cbdc9..0c6d8dd 100644 --- a/supervised_finetuning/sft_coder.py +++ b/supervised_finetuning/sft_coder.py @@ -15,7 +15,7 @@ if __name__ == "__main__": process_dataset( dataset_dict=dataset, - output_subdir="Ling-Coder-SFT", + output_subdir="Ling-Coder-sft", process_func=process_func ) \ No newline at end of file diff --git a/supervised_finetuning/sft_magpie-pro-300k.py b/supervised_finetuning/sft_magpie-pro-300k.py index 4e92544..50d24f2 100644 --- a/supervised_finetuning/sft_magpie-pro-300k.py +++ b/supervised_finetuning/sft_magpie-pro-300k.py @@ -24,7 +24,7 @@ if __name__ == "__main__": dataset = load_dataset("HuggingFaceTB/Magpie-Pro-300K-Filtered-H4") process_dataset( dataset_dict=dataset, - output_subdir="belle-sft", + output_subdir="Magpie-Pro-300K-sft", process_func=process_func, - split_name="train-sft" + split_name="train_sft", ) \ No newline at end of file