feat(chinese-wiki., english-wiki.): 添加 max_chunk_size 参数

2025-07-14 13:35:06 +08:00
parent 820ddf3fc8
commit 1c132826c0
2 changed files with 4 additions and 2 deletions
@@ -4,5 +4,6 @@ if __name__ == "__main__":
    process_dataset(
        dataset_name="wikimedia/wikipedia",
        output_subdir="chinese-wiki",
-        dataset_config="zh"
+        dataset_config="zh",
+        max_chunk_size=5,
    )