refactor(project): 将 utils 和 tokenizer 模块移至 modules 目录

supervised_finetuning/sft_belle.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 

supervised_finetuning/sft_chinese_instruct.py

@@ -1,6 +1,6 @@
 from datasets import DatasetDict
 from datasets import load_dataset, concatenate_datasets
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 def replace_seg(query:str, response:str) -> str:

supervised_finetuning/sft_coder.py

@@ -1,6 +1,6 @@
 # inclusionAI/Ling-Coder-SFT
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 def process_func(input_dict: dict) -> dict:

supervised_finetuning/sft_magpie-pro-300k.py

@@ -1,6 +1,6 @@
 # HuggingFaceTB/Magpie-Pro-300K-Filtered-H4
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 
 def process_func(input_dict: dict):