diff --git a/chinese-wiki.py b/chinese-wiki.py index e0afb99..dc979a7 100644 --- a/chinese-wiki.py +++ b/chinese-wiki.py @@ -2,6 +2,7 @@ from utils import process_dataset if __name__ == "__main__": process_dataset( - dataset_name="Blaze7451/Wiki-zh-20250601", - output_subdir="chinese-wiki" + dataset_name="wikimedia/wikipedia", + output_subdir="chinese-wiki", + dataset_config="zh" ) \ No newline at end of file diff --git a/utils.py b/utils.py index 1e15b6a..610ab00 100644 --- a/utils.py +++ b/utils.py @@ -24,7 +24,7 @@ def process_dataset( normalization_func=comprehensive_normalization ): - dataset_dict = load_dataset(dataset_name, dataset_config) if dataset_config else load_dataset(dataset_name) + dataset_dict = load_dataset(dataset_name, dataset_config) train_dataset = dataset_dict[split_name] total_samples = len(train_dataset)