diff --git a/belle_sft.py b/belle_sft.py new file mode 100644 index 0000000..c4e1258 --- /dev/null +++ b/belle_sft.py @@ -0,0 +1,10 @@ +from datasets import load_dataset +from utils import pre_tarin_process + +if __name__ == "__main__": + dataset = load_dataset("BelleGroup/train_3.5M_CN") + # pre_tarin_process( + # dataset_dict=dataset, + # output_subdir="belle_sft", + # max_chunk_size=5, + # ) \ No newline at end of file diff --git a/chinese-c4.py b/chinese-c4.py index 9344039..154f342 100644 --- a/chinese-c4.py +++ b/chinese-c4.py @@ -1,9 +1,9 @@ from datasets import load_dataset -from utils import process_dataset +from utils import pre_tarin_process if __name__ == "__main__": dataset = load_dataset("shjwudp/chinese-c4") - process_dataset( + pre_tarin_process( dataset_dict=dataset, output_subdir="chinese-c4" ) \ No newline at end of file diff --git a/chinese-cosmopedia.py b/chinese-cosmopedia.py index 94d7ba0..d22cfa0 100644 --- a/chinese-cosmopedia.py +++ b/chinese-cosmopedia.py @@ -1,5 +1,5 @@ from datasets import load_dataset -from utils import process_dataset +from utils import pre_tarin_process if __name__ == "__main__": max_chunk_num = 10 @@ -10,7 +10,7 @@ if __name__ == "__main__": data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]} ) - process_dataset( + pre_tarin_process( dataset_dict=dataset, output_subdir="chinese-wiki", max_chunk_num=max_chunk_num, diff --git a/english-fineweb.py b/english-fineweb.py index e71a42d..855741c 100644 --- a/english-fineweb.py +++ b/english-fineweb.py @@ -1,9 +1,9 @@ from datasets import load_dataset -from utils import process_dataset +from utils import pre_tarin_process if __name__ == "__main__": dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT") - process_dataset( + pre_tarin_process( dataset_dict=dataset, output_subdir="english-fineweb", ) \ No newline at end of file diff --git a/english-wiki.py b/english-wiki.py index e15f265..c7e180d 100644 --- a/english-wiki.py +++ b/english-wiki.py @@ -1,9 +1,9 @@ from datasets import load_dataset -from utils import process_dataset +from utils import pre_tarin_process if __name__ == "__main__": dataset = load_dataset("Blaze7451/enwiki_structured_content") - process_dataset( + pre_tarin_process( dataset_dict=dataset, output_subdir="english-wiki", max_chunk_size=5, diff --git a/to_ids.py b/pre_train_file.py similarity index 68% rename from to_ids.py rename to pre_train_file.py index ee0adf7..1822141 100644 --- a/to_ids.py +++ b/pre_train_file.py @@ -8,10 +8,10 @@ def processor(intput_str: str): if __name__ == "__main__": tokenizer = BpeTokenizer("tokenizer.json") base_dir = [ - # os.path.join("dataset", "chinese-c4"), - # os.path.join("dataset", "english-fineweb"), - # os.path.join("dataset", "english-wiki"), - # os.path.join("dataset", "chinese-wiki"), + os.path.join("dataset", "chinese-c4"), + os.path.join("dataset", "english-fineweb"), + os.path.join("dataset", "english-wiki"), + os.path.join("dataset", "chinese-wiki"), ] base_out_dir = "pkl_output" files = [] diff --git a/sft_file.py b/sft_file.py new file mode 100644 index 0000000..e69de29 diff --git a/utils.py b/utils.py index d62d4ac..4abf9ea 100644 --- a/utils.py +++ b/utils.py @@ -57,7 +57,7 @@ def dump_pkl_files( tensor = torch.cat(arrows) pkl.dump(tensor, f) -def process_dataset( +def pre_tarin_process( dataset_dict: DatasetDict, output_subdir: str, max_chunk_num: int = None, @@ -90,3 +90,10 @@ def process_dataset( f.write(json.dumps(json_line, ensure_ascii=False) + "\n") print(f"Saved text chunk {i} to {output_path}") + + +def sft_process( + datset_dict: DatasetDict, + output_subdir: str, +): + pass \ No newline at end of file