feat: add sft scripts for Magicoder-Evol-Instruct-110K and alpaca-gpt4-data

- sft_magicoder.py: export ise-uiuc/Magicoder-Evol-Instruct-110K
- sft_alpaca_gpt4.py: export llm-wizard/alpaca-gpt4-data with instruction+input
- remove deprecated SFT scripts

scripts/pre_train/chinese-c4.py

@@ -1,10 +0,0 @@
-from datasets import load_dataset
-from pipeline import export_dataset
-
-if __name__ == "__main__":
-    dataset = load_dataset("shjwudp/chinese-c4")
-    export_dataset(
-        dataset=dataset["train"],
-        output_dir="./dataset",
-        output_prefix="chinese-c4-pretrain",
-    )

scripts/pre_train/english-fineweb.py

@@ -1,10 +0,0 @@
-from datasets import load_dataset
-from pipeline import export_dataset
-
-if __name__ == "__main__":
-    dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
-    export_dataset(
-        dataset=dataset["train"],
-        output_dir="./dataset",
-        output_prefix="english-fineweb-pretrain",
-    )

scripts/supervised_finetuning/sft_alpaca_gpt4.py

@@ -0,0 +1,22 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict):
+    instruction = input_dict["instruction"]
+    inp = input_dict.get("input", "")
+    if inp:
+        query = instruction + "\n" + inp
+    else:
+        query = instruction
+    return {"query": query, "response": input_dict["output"]}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("llm-wizard/alpaca-gpt4-data")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="alpaca-gpt4-data",
+        process_func=process_func,
+    )

scripts/supervised_finetuning/sft_belle.py

@@ -1,23 +0,0 @@
-from datasets import load_dataset
-from pipeline import export_dataset
-
-
-def process_func(input_dict: dict):
-    conversations = input_dict["conversations"]
-    n = len(conversations) // 2
-    examples = []
-    for i in range(n):
-        user_msg = conversations[2 * i]["value"]
-        assistant_msg = conversations[2 * i + 1]["value"]
-        examples.append({"query": user_msg, "response": assistant_msg})
-    return examples
-
-
-if __name__ == "__main__":
-    dataset = load_dataset("BelleGroup/train_3.5M_CN")
-    export_dataset(
-        dataset=dataset["train"],
-        output_dir="./dataset",
-        output_prefix="belle-sft",
-        process_func=process_func,
-    )

scripts/supervised_finetuning/sft_chinese_instruct.py

@@ -1,45 +0,0 @@
-from datasets import load_dataset, concatenate_datasets
-from pipeline import export_dataset, TextNormalizer
-
-normalizer = TextNormalizer()
-
-
-def process_func(input_dict: dict):
-    query = input_dict["prompt"] if input_dict["prompt"] else ""
-    resp = input_dict["response"] if input_dict["response"] else ""
-    return {
-        "query": normalizer.normalize(query),
-        "response": normalizer.normalize(resp),
-    }
-
-
-if __name__ == "__main__":
-    all_data = [
-        "stem_zh",
-        "infinity-instruct",
-        "firefly",
-        "magpie",
-        "dpsk-r1-distil",
-        "coig-cqia",
-        "disc-law",
-        "neo_sft_phase2",
-        "chinese-medical",
-        "chinese-reasoning-distil",
-        "psycho-10k-dpsk-r1",
-        "sof-c-zh",
-        "industryinstruction",
-        "Chinese-QA-AFAF",
-    ]
-
-    dataset_list = []
-    for subset in all_data:
-        ds = load_dataset("Mxode/Chinese-Instruct", name=subset)
-        dataset_list.append(ds["train"])
-
-    combined_dataset = concatenate_datasets(dataset_list)
-    export_dataset(
-        dataset=combined_dataset,
-        output_dir="./dataset",
-        output_prefix="chinese-instruct-sft",
-        process_func=process_func,
-    )

scripts/supervised_finetuning/sft_coder.py

@@ -1,19 +0,0 @@
-from datasets import load_dataset
-from pipeline import export_dataset
-
-
-def process_func(input_dict: dict) -> dict:
-    msg = input_dict["messages"]
-    query = msg[0]["content"]
-    history = msg[1]["content"]
-    return {"query": query, "response": history}
-
-
-if __name__ == "__main__":
-    dataset = load_dataset("inclusionAI/Ling-Coder-SFT")
-    export_dataset(
-        dataset=dataset["train"],
-        output_dir="./dataset",
-        output_prefix="Ling-Coder-sft",
-        process_func=process_func,
-    )

scripts/supervised_finetuning/sft_magicoder.py

@@ -0,0 +1,16 @@
+from datasets import load_dataset
+from pipeline import export_dataset
+
+
+def process_func(input_dict: dict):
+    return {"query": input_dict["instruction"], "response": input_dict["response"]}
+
+
+if __name__ == "__main__":
+    dataset = load_dataset("ise-uiuc/Magicoder-Evol-Instruct-110K")
+    export_dataset(
+        dataset=dataset["train"],
+        output_dir="./dataset",
+        output_prefix="Magicoder-Evol-Instruct-110K",
+        process_func=process_func,
+    )

scripts/supervised_finetuning/sft_magpie-pro-300k.py

@@ -1,24 +0,0 @@
-from datasets import load_dataset
-from pipeline import export_dataset
-
-
-def process_func(input_dict: dict):
-    conversations = input_dict["conversations"]
-    assert len(conversations) % 2 == 0
-    n = len(conversations) // 2
-    examples = []
-    for i in range(n):
-        user_msg = conversations[2 * i]["value"]
-        assistant_msg = conversations[2 * i + 1]["value"]
-        examples.append({"query": user_msg, "response": assistant_msg})
-    return examples
-
-
-if __name__ == "__main__":
-    dataset = load_dataset("HuggingFaceTB/Magpie-Pro-300K-Filtered-H4")
-    export_dataset(
-        dataset=dataset["train_sft"],
-        output_dir="./dataset",
-        output_prefix="Magpie-Pro-300K-sft",
-        process_func=process_func,
-    )