refactor(dataset): 更新数据集加载路径和处理代码

2025-07-15 14:06:18 +08:00
parent e77fb56c45
commit 6edc31ce2e
2 changed files with 1 additions and 2 deletions
@@ -7,7 +7,7 @@ if __name__ == "__main__":
    
    dataset = load_dataset(
        "opencsg/chinese-cosmopedia", 
-        data_files=[f"0000{i}.parquet" for i in range(5)]
+        data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]}
    )
    
    process_dataset(
@@ -66,7 +66,6 @@ def process_dataset(
    column_name: str = "text",
    normalization_func=comprehensive_normalization,
 ):
-
    train_dataset = dataset_dict[split_name]
    total_samples = len(train_dataset)
    num_chunks = (total_samples // chunk_size) + 1