fix: 修复特殊token 问题

2026-04-02 16:16:02 +08:00
parent e01ec081b3
commit f44ad6912e
28 changed files with 334 additions and 163 deletions
@@ -4,7 +4,7 @@ from pipeline import export_dataset
 if __name__ == "__main__":
    dataset = load_dataset(
        "opencsg/chinese-cosmopedia",
-        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]}
+        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]},
    )
    export_dataset(
        dataset=dataset["train"],