refactor(sft_file, utils): 重构数据处理流程并支持新数据集

2025-07-21 15:11:57 +08:00
parent 56b64ab8c3
commit b6c02c04b2
2 changed files with 42 additions and 46 deletions
@@ -5,7 +5,8 @@ import os
 if __name__ == "__main__":
    tokenizer = BpeTokenizer("tokenizer.json")
    base_dir = [
-        os.path.join("dataset", "belle-sft"),
+        # os.path.join("dataset", "belle-sft"),
+        os.path.join("dataset", "chinese-instruct")
    ]
    base_out_dir = "pkl_output"
    files = []