feat(chinese-wiki., english-wiki.): 添加 max_chunk_size 参数

2025-07-14 13:35:06 +08:00
parent 820ddf3fc8
commit 1c132826c0
2 changed files with 4 additions and 2 deletions
@@ -4,5 +4,6 @@ if __name__ == "__main__":
    process_dataset(
        dataset_name="wikimedia/wikipedia",
        output_subdir="chinese-wiki",
-        dataset_config="zh"
+        dataset_config="zh",
+        max_chunk_size=5,
    )
@@ -3,5 +3,6 @@ from utils import process_dataset
 if __name__ == "__main__":
    process_dataset(
        dataset_name="Blaze7451/enwiki_structured_content",
-        output_subdir="english-wiki"
+        output_subdir="english-wiki",
+        max_chunk_size=5,
    )