fix: 修复特殊token 问题

scripts/cache_h5.py

@@ -7,6 +7,7 @@ Usage:
     python scripts/cache_h5.py sft ./dataset/belle-sft --pack-size 4096 --strategy alpaca
     python scripts/cache_h5.py sft ./dataset/Ling-Coder-sft --tokenizer ./my_tokenizer.json
 """
+
 import argparse
 import os
 
@@ -18,16 +19,34 @@ def main():
     parser = argparse.ArgumentParser(description="JSONL -> H5 cache")
     parser.add_argument("type", choices=["pt", "sft", "dpo"], help="Processor type")
     parser.add_argument("input_dir", help="Directory containing JSONL files")
-    parser.add_argument("-o", "--output-dir", default=None,
-                        help="H5 output dir (default: <input_dir>/cached)")
-    parser.add_argument("-t", "--tokenizer", default="./tokenizer.json",
-                        help="Tokenizer path (default: ./tokenizer.json)")
-    parser.add_argument("-s", "--strategy", default=None,
-                        help="Prompt strategy: chatml, alpaca (default: chatml)")
-    parser.add_argument("-p", "--pack-size", type=int, default=-1,
-                        help="Pack size, <=0 to disable (default: -1)")
-    parser.add_argument("--pad-value", type=int, default=1,
-                        help="Padding value (default: 1)")
+    parser.add_argument(
+        "-o",
+        "--output-dir",
+        default=None,
+        help="H5 output dir (default: <input_dir>/cached)",
+    )
+    parser.add_argument(
+        "-t",
+        "--tokenizer",
+        default="./tokenizer.json",
+        help="Tokenizer path (default: ./tokenizer.json)",
+    )
+    parser.add_argument(
+        "-s",
+        "--strategy",
+        default=None,
+        help="Prompt strategy: chatml, alpaca (default: chatml)",
+    )
+    parser.add_argument(
+        "-p",
+        "--pack-size",
+        type=int,
+        default=-1,
+        help="Pack size, <=0 to disable (default: -1)",
+    )
+    parser.add_argument(
+        "--pad-value", type=int, default=1, help="Padding value (default: 1)"
+    )
     args = parser.parse_args()
 
     jsonl_files = IOHandler.fetch_files(args.input_dir, suffix=".jsonl")

scripts/pre_train/chinese-cosmopedia.py

@@ -4,7 +4,7 @@ from pipeline import export_dataset
 if __name__ == "__main__":
     dataset = load_dataset(
         "opencsg/chinese-cosmopedia",
-        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]}
+        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]},
     )
     export_dataset(
         dataset=dataset["train"],

scripts/supervised_finetuning/sft_chinese_instruct.py

@@ -7,14 +7,28 @@ normalizer = TextNormalizer()
 def process_func(input_dict: dict):
     query = input_dict["prompt"] if input_dict["prompt"] else ""
     resp = input_dict["response"] if input_dict["response"] else ""
-    return {"query": normalizer.normalize(query), "response": normalizer.normalize(resp)}
+    return {
+        "query": normalizer.normalize(query),
+        "response": normalizer.normalize(resp),
+    }
 
 
 if __name__ == "__main__":
     all_data = [
-        'stem_zh', 'infinity-instruct', 'firefly', 'magpie', 'dpsk-r1-distil',
-        'coig-cqia', 'disc-law', 'neo_sft_phase2', 'chinese-medical', 'chinese-reasoning-distil',
-        'psycho-10k-dpsk-r1', 'sof-c-zh', 'industryinstruction', 'Chinese-QA-AFAF',
+        "stem_zh",
+        "infinity-instruct",
+        "firefly",
+        "magpie",
+        "dpsk-r1-distil",
+        "coig-cqia",
+        "disc-law",
+        "neo_sft_phase2",
+        "chinese-medical",
+        "chinese-reasoning-distil",
+        "psycho-10k-dpsk-r1",
+        "sof-c-zh",
+        "industryinstruction",
+        "Chinese-QA-AFAF",
     ]
 
     dataset_list = []