refactor: 修改项目结构

scripts/pre_train/chinese-c4.py

@@ -0,0 +1,10 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("shjwudp/chinese-c4")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="chinese-c4-pretrain",
+    )

scripts/pre_train/chinese-cosmopedia.py

@@ -0,0 +1,14 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset(
+        "opencsg/chinese-cosmopedia",
+        data_files={"train": [f"data/000{i:02d}.parquet" for i in range(25)]}
+    )
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="chinese-cosmopedia-pretrain",
+        chunk_size=1_000_000,
+    )

scripts/pre_train/english-fineweb.py

@@ -0,0 +1,10 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="english-fineweb-pretrain",
+    )

scripts/pre_train/english-wiki.py

@@ -0,0 +1,11 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+if __name__ == "__main__":
+    dataset = load_dataset("Blaze7451/enwiki_structured_content")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="english-wiki-pretrain",
+        max_chunks=5,
+    )

scripts/reforce_learning/dpp_chinese_dpo_pairs.py

@@ -0,0 +1,20 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict):
+    return {
+        "prompt": input_dict["prompt"],
+        "chosen": input_dict["chosen"],
+        "rejected": input_dict["rejected"],
+    }
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("wenbopan/Chinese-dpo-pairs")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="Chinese-dpo-pairs",
+        process_func=process_func,
+    )

scripts/supervised_finetuning/sft_belle.py

@@ -0,0 +1,23 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict):
+    conversations = input_dict["conversations"]
+    n = len(conversations) // 2
+    examples = []
+    for i in range(n):
+        user_msg = conversations[2 * i]["value"]
+        assistant_msg = conversations[2 * i + 1]["value"]
+        examples.append({"query": user_msg, "response": assistant_msg})
+    return examples
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("BelleGroup/train_3.5M_CN")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="belle-sft",
+        process_func=process_func,
+    )

scripts/supervised_finetuning/sft_chinese_instruct.py

@@ -0,0 +1,31 @@
+from datasets import load_dataset, concatenate_datasets
+from pipeline import export_dataset, TextNormalizer
+
+normalizer = TextNormalizer()
+
+
+def process_func(input_dict: dict):
+    query = input_dict["prompt"] if input_dict["prompt"] else ""
+    resp = input_dict["response"] if input_dict["response"] else ""
+    return {"query": normalizer.normalize(query), "response": normalizer.normalize(resp)}
+
+
+if __name__ == "__main__":
+    all_data = [
+        'stem_zh', 'infinity-instruct', 'firefly', 'magpie', 'dpsk-r1-distil',
+        'coig-cqia', 'disc-law', 'neo_sft_phase2', 'chinese-medical', 'chinese-reasoning-distil',
+        'psycho-10k-dpsk-r1', 'sof-c-zh', 'industryinstruction', 'Chinese-QA-AFAF',
+    ]
+
+    datasets = []
+    for subset in all_data:
+        ds = load_dataset("Mxode/Chinese-Instruct", name=subset)
+        datasets.append(ds["train"])
+
+    combined_dataset = concatenate_datasets(datasets)
+    export_dataset(
+        dataset=combined_dataset,
+        output_dir="./dataset",
+        output_prefix="chinese-instruct-sft",
+        process_func=process_func,
+    )

scripts/supervised_finetuning/sft_coder.py

@@ -0,0 +1,19 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict) -> dict:
+    msg = input_dict["messages"]
+    query = msg[0]["content"]
+    history = msg[1]["content"]
+    return {"query": query, "response": history}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("inclusionAI/Ling-Coder-SFT")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="Ling-Coder-sft",
+        process_func=process_func,
+    )

scripts/supervised_finetuning/sft_firefly-1.1m-rephrased.py

@@ -0,0 +1,16 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict):
+    return {"query": input_dict["instruction"], "response": input_dict["output"]}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("Mxode/Firefly-1.1M-Rephrased")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="Firefly-1.1M-Rephrased",
+        process_func=process_func,
+    )

scripts/supervised_finetuning/sft_magpie-pro-300k.py

@@ -0,0 +1,24 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict):
+    conversations = input_dict["conversations"]
+    assert len(conversations) % 2 == 0
+    n = len(conversations) // 2
+    examples = []
+    for i in range(n):
+        user_msg = conversations[2 * i]["value"]
+        assistant_msg = conversations[2 * i + 1]["value"]
+        examples.append({"query": user_msg, "response": assistant_msg})
+    return examples
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("HuggingFaceTB/Magpie-Pro-300K-Filtered-H4")
+    export_dataset(
+        dataset=dataset["train_sft"],
+        output_dir="./dataset",
+        output_prefix="Magpie-Pro-300K-sft",
+        process_func=process_func,
+    )