From 092921f4015753eb621c5425d8f99d285929f057 Mon Sep 17 00:00:00 2001 From: ViperEkura <3081035982@qq.com> Date: Fri, 1 Aug 2025 20:59:17 +0800 Subject: [PATCH] =?UTF-8?q?refactor(project):=20=E5=B0=86=20utils=20?= =?UTF-8?q?=E5=92=8C=20tokenizer=20=E6=A8=A1=E5=9D=97=E7=A7=BB=E8=87=B3=20?= =?UTF-8?q?modules=20=E7=9B=AE=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dump_pt_file.py | 4 ++-- dump_sft_file.py | 4 ++-- tokenizer.py => modules/tokenizer.py | 0 utils.py => modules/utils.py | 0 pre_train/chinese-c4.py | 2 +- pre_train/chinese-cosmopedia.py | 2 +- pre_train/english-fineweb.py | 2 +- pre_train/english-wiki.py | 2 +- supervised_finetuning/sft_belle.py | 2 +- supervised_finetuning/sft_chinese_instruct.py | 2 +- supervised_finetuning/sft_coder.py | 2 +- supervised_finetuning/sft_magpie-pro-300k.py | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) rename tokenizer.py => modules/tokenizer.py (100%) rename utils.py => modules/utils.py (100%) diff --git a/dump_pt_file.py b/dump_pt_file.py index c6e17d5..f924aa1 100644 --- a/dump_pt_file.py +++ b/dump_pt_file.py @@ -1,5 +1,5 @@ -from utils import dump_pkl_files, fetch_files -from tokenizer import BpeTokenizer +from modules.utils import dump_pkl_files, fetch_files +from modules.tokenizer import BpeTokenizer import torch import os diff --git a/dump_sft_file.py b/dump_sft_file.py index 6b11738..1a5db3c 100644 --- a/dump_sft_file.py +++ b/dump_sft_file.py @@ -1,5 +1,5 @@ -from utils import dump_pkl_files, fetch_files -from tokenizer import BpeTokenizer +from modules.utils import dump_pkl_files, fetch_files +from modules.tokenizer import BpeTokenizer import torch import os diff --git a/tokenizer.py b/modules/tokenizer.py similarity index 100% rename from tokenizer.py rename to modules/tokenizer.py diff --git a/utils.py b/modules/utils.py similarity index 100% rename from utils.py rename to modules/utils.py diff --git a/pre_train/chinese-c4.py b/pre_train/chinese-c4.py index 9344039..1c3f2dc 100644 --- a/pre_train/chinese-c4.py +++ b/pre_train/chinese-c4.py @@ -1,5 +1,5 @@ from datasets import load_dataset -from utils import process_dataset +from modules.utils import process_dataset if __name__ == "__main__": dataset = load_dataset("shjwudp/chinese-c4") diff --git a/pre_train/chinese-cosmopedia.py b/pre_train/chinese-cosmopedia.py index 94d7ba0..f4a88e5 100644 --- a/pre_train/chinese-cosmopedia.py +++ b/pre_train/chinese-cosmopedia.py @@ -1,5 +1,5 @@ from datasets import load_dataset -from utils import process_dataset +from modules.utils import process_dataset if __name__ == "__main__": max_chunk_num = 10 diff --git a/pre_train/english-fineweb.py b/pre_train/english-fineweb.py index e71a42d..ecd5fc3 100644 --- a/pre_train/english-fineweb.py +++ b/pre_train/english-fineweb.py @@ -1,5 +1,5 @@ from datasets import load_dataset -from utils import process_dataset +from modules.utils import process_dataset if __name__ == "__main__": dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT") diff --git a/pre_train/english-wiki.py b/pre_train/english-wiki.py index e15f265..2f3f43d 100644 --- a/pre_train/english-wiki.py +++ b/pre_train/english-wiki.py @@ -1,5 +1,5 @@ from datasets import load_dataset -from utils import process_dataset +from modules.utils import process_dataset if __name__ == "__main__": dataset = load_dataset("Blaze7451/enwiki_structured_content") diff --git a/supervised_finetuning/sft_belle.py b/supervised_finetuning/sft_belle.py index f86b04d..b01ff1f 100644 --- a/supervised_finetuning/sft_belle.py +++ b/supervised_finetuning/sft_belle.py @@ -1,5 +1,5 @@ from datasets import load_dataset -from utils import process_dataset +from modules.utils import process_dataset diff --git a/supervised_finetuning/sft_chinese_instruct.py b/supervised_finetuning/sft_chinese_instruct.py index 8a2d613..6bda02f 100644 --- a/supervised_finetuning/sft_chinese_instruct.py +++ b/supervised_finetuning/sft_chinese_instruct.py @@ -1,6 +1,6 @@ from datasets import DatasetDict from datasets import load_dataset, concatenate_datasets -from utils import process_dataset +from modules.utils import process_dataset def replace_seg(query:str, response:str) -> str: diff --git a/supervised_finetuning/sft_coder.py b/supervised_finetuning/sft_coder.py index 3099f16..c4cbdc9 100644 --- a/supervised_finetuning/sft_coder.py +++ b/supervised_finetuning/sft_coder.py @@ -1,6 +1,6 @@ # inclusionAI/Ling-Coder-SFT from datasets import load_dataset -from utils import process_dataset +from modules.utils import process_dataset def process_func(input_dict: dict) -> dict: diff --git a/supervised_finetuning/sft_magpie-pro-300k.py b/supervised_finetuning/sft_magpie-pro-300k.py index 897852c..4e92544 100644 --- a/supervised_finetuning/sft_magpie-pro-300k.py +++ b/supervised_finetuning/sft_magpie-pro-300k.py @@ -1,6 +1,6 @@ # HuggingFaceTB/Magpie-Pro-300K-Filtered-H4 from datasets import load_dataset -from utils import process_dataset +from modules.utils import process_dataset def process_func(input_dict: dict):