diff --git a/chinese-wiki.py b/chinese-wiki.py index dc979a7..dca111f 100644 --- a/chinese-wiki.py +++ b/chinese-wiki.py @@ -4,5 +4,6 @@ if __name__ == "__main__": process_dataset( dataset_name="wikimedia/wikipedia", output_subdir="chinese-wiki", - dataset_config="zh" + dataset_config="zh", + max_chunk_size=5, ) \ No newline at end of file diff --git a/english-wiki.py b/english-wiki.py index d9f7ae1..cdeca88 100644 --- a/english-wiki.py +++ b/english-wiki.py @@ -3,5 +3,6 @@ from utils import process_dataset if __name__ == "__main__": process_dataset( dataset_name="Blaze7451/enwiki_structured_content", - output_subdir="english-wiki" + output_subdir="english-wiki", + max_chunk_size=5, ) \ No newline at end of file