refactor(project): 将 utils 和 tokenizer 模块移至 modules 目录

pre_train/chinese-c4.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     dataset = load_dataset("shjwudp/chinese-c4")

pre_train/chinese-cosmopedia.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     max_chunk_num = 10

pre_train/english-fineweb.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")

pre_train/english-wiki.py

@@ -1,5 +1,5 @@
 from datasets import load_dataset
-from utils import process_dataset
+from modules.utils import process_dataset
 
 if __name__ == "__main__":
     dataset = load_dataset("Blaze7451/enwiki_structured_content")