Add max_chunks=18 to dolma-30b export, remove max_chunks=5 from english-wiki, rename project to datapipline

2026-07-06 08:53:18 +08:00
parent 900cd91798
commit fec37545e2
3 changed files with 12 additions and 2 deletions
@@ -7,5 +7,4 @@ if __name__ == "__main__":
        dataset=dataset["train"],
        output_dir="./dataset",
        output_prefix="english-wiki-pretrain",
-        max_chunks=5,
    )