refactor(to_ids): 注释掉中文维基百科数据集的处理

2025-07-15 07:01:26 +08:00
parent 95a3568328
commit 1c7e49a631
1 changed files with 1 additions and 1 deletions
@@ -11,7 +11,7 @@ if __name__ == "__main__":
        # os.path.join("dataset", "chinese-c4"),
        # os.path.join("dataset", "english-fineweb"),
        # os.path.join("dataset", "english-wiki"),
-        os.path.join("dataset", "chinese-wiki"),
+        # os.path.join("dataset", "chinese-wiki"),
    ]
    base_out_dir = "pkl_output"
    files = []