移除缓存脚本中的 data_files 和 max_chunks 限制参数

2026-07-26 10:57:30 +08:00
parent 33c8720d69
commit e999629098
2 changed files with 0 additions and 2 deletions
@@ -4,7 +4,6 @@ from pipeline import export_dataset
 if __name__ == "__main__":
    dataset = load_dataset(
        "opencsg/chinese-cosmopedia",
-        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]},
    )
    export_dataset(
        dataset=dataset["train"],
@@ -7,5 +7,4 @@ if __name__ == "__main__":
        dataset=dataset["train"],
        output_dir="./dataset",
        output_prefix="english-dolma-30b-pretrain",
-        max_chunks=18,
    )