refactor(data_processing): 重构数据处理脚本并添加 fetch_folders 函数

dump_sft_file.py

@@ -1,19 +1,14 @@
-from modules.utils import dump_pkl_files, fetch_files, get_sft_processor
+from modules.utils import dump_pkl_files, fetch_files, fetch_folders, get_sft_processor
 from modules.tokenizer import BpeTokenizer
-import os
-
 
 if __name__ == "__main__":
     tokenizer = BpeTokenizer("tokenizer.json")
-    base_dir = [
-        os.path.join("dataset", "Ling-Coder-SFT"),
-        os.path.join("dataset", "chinese-instruct")
-    ]
+    base_dir = fetch_folders("dataset")
+    
     base_out_dir = "pkl_output"
     files = []
     for dir_path in base_dir:
         files.extend(fetch_files(dir_path))
         
     processor = get_sft_processor(tokenizer)
-    
     dump_pkl_files(files, base_out_dir,processor, ["sequence", "mask"])