diff --git a/scripts/pre_train/chinese-cosmopedia.py b/scripts/pre_train/chinese-cosmopedia.py index 8a1e769..01f64b3 100644 --- a/scripts/pre_train/chinese-cosmopedia.py +++ b/scripts/pre_train/chinese-cosmopedia.py @@ -4,7 +4,6 @@ from pipeline import export_dataset if __name__ == "__main__": dataset = load_dataset( "opencsg/chinese-cosmopedia", - data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]}, ) export_dataset( dataset=dataset["train"], diff --git a/scripts/pre_train/english-dolma.py b/scripts/pre_train/english-dolma.py index 07ef4c7..7e82b3c 100644 --- a/scripts/pre_train/english-dolma.py +++ b/scripts/pre_train/english-dolma.py @@ -7,5 +7,4 @@ if __name__ == "__main__": dataset=dataset["train"], output_dir="./dataset", output_prefix="english-dolma-30b-pretrain", - max_chunks=18, )