refactor(pretrain): 更新预训练数据集输出目录名称

2025-08-02 17:29:21 +08:00
parent 931403a713
commit 241e24c400
7 changed files with 8 additions and 8 deletions
@@ -5,5 +5,5 @@ if __name__ == "__main__":
    dataset = load_dataset("shjwudp/chinese-c4")
    process_dataset(
        dataset_dict=dataset,
-        output_subdir="chinese-c4"
+        output_subdir="chinese-c4-pretrain"
    )
@@ -12,7 +12,7 @@ if __name__ == "__main__":
    
    process_dataset(
        dataset_dict=dataset,
-        output_subdir="chinese-wiki",
+        output_subdir="chinese-wiki-pretrain",
        max_chunk_num=max_chunk_num,
        chunk_size=chunk_size,
    )
@@ -5,5 +5,5 @@ if __name__ == "__main__":
    dataset = load_dataset("HuggingFaceFW/fineweb", "sample-10BT")
    process_dataset(
        dataset_dict=dataset,
-        output_subdir="english-fineweb",
+        output_subdir="english-fineweb-pretrain",
    )
@@ -5,6 +5,6 @@ if __name__ == "__main__":
    dataset = load_dataset("Blaze7451/enwiki_structured_content")
    process_dataset(
        dataset_dict=dataset,
-        output_subdir="english-wiki",
+        output_subdir="english-wiki-pretrain",
        max_chunk_size=5,
    )