diff --git a/scripts/pre_train/ultra-fineweb-l3-en-qa-synthetic.py b/scripts/pre_train/ultra-fineweb-l3-en-qa-synthetic.py new file mode 100644 index 0000000..eab46eb --- /dev/null +++ b/scripts/pre_train/ultra-fineweb-l3-en-qa-synthetic.py @@ -0,0 +1,20 @@ +from datasets import load_dataset +from pipeline import export_dataset + + +def process_func(input_dict: dict): + return {"text": input_dict["content"]} + + +if __name__ == "__main__": + dataset = load_dataset( + "openbmb/Ultra-FineWeb-L3", + "Ultra-FineWeb-L3-en-QA-Synthetic", + split="train", + ) + export_dataset( + dataset=dataset, + output_dir="./dataset", + output_prefix="ultra-fineweb-l3-en-qa-synthetic-pretrain", + process_func=process_func, + ) diff --git a/scripts/pre_train/ultra-fineweb-l3-zh-qa-synthetic.py b/scripts/pre_train/ultra-fineweb-l3-zh-qa-synthetic.py new file mode 100644 index 0000000..e7b005f --- /dev/null +++ b/scripts/pre_train/ultra-fineweb-l3-zh-qa-synthetic.py @@ -0,0 +1,20 @@ +from datasets import load_dataset +from pipeline import export_dataset + + +def process_func(input_dict: dict): + return {"text": input_dict["content"]} + + +if __name__ == "__main__": + dataset = load_dataset( + "openbmb/Ultra-FineWeb-L3", + "Ultra-FineWeb-L3-zh-QA-Synthetic", + split="train", + ) + export_dataset( + dataset=dataset, + output_dir="./dataset", + output_prefix="ultra-fineweb-l3-zh-qa-synthetic-pretrain", + process_func=process_func, + )