refactor(project): 将 utils 和 tokenizer 模块移至 modules 目录

dump_pt_file.py

@@ -1,5 +1,5 @@
-from utils import dump_pkl_files, fetch_files
-from tokenizer import BpeTokenizer
+from modules.utils import dump_pkl_files, fetch_files
+from modules.tokenizer import BpeTokenizer
 import torch
 import os
 

dump_sft_file.py

@@ -1,5 +1,5 @@
-from utils import dump_pkl_files, fetch_files
-from tokenizer import BpeTokenizer
+from modules.utils import dump_pkl_files, fetch_files
+from modules.tokenizer import BpeTokenizer
 import torch
 import os
 

pre_train/chinese-c4.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     dataset = load_dataset("shjwudp/chinese-c4")

pre_train/chinese-cosmopedia.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     max_chunk_num = 10

pre_train/english-fineweb.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")

pre_train/english-wiki.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     dataset = load_dataset("Blaze7451/enwiki_structured_content")

supervised_finetuning/sft_belle.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 

supervised_finetuning/sft_chinese_instruct.py

@@ -1,6 +1,6 @@
 from datasets import DatasetDict
 from datasets import load_dataset, concatenate_datasets
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 def replace_seg(query:str, response:str) -> str:

supervised_finetuning/sft_coder.py

@@ -1,6 +1,6 @@
 # inclusionAI/Ling-Coder-SFT
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 def process_func(input_dict: dict) -> dict:

supervised_finetuning/sft_magpie-pro-300k.py

@@ -1,6 +1,6 @@
 # HuggingFaceTB/Magpie-Pro-300K-Filtered-H4
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 def process_func(input_dict: dict):