refactor(dataset): 更新数据集加载路径和处理代码

2025-07-15 14:06:18 +08:00
parent e77fb56c45
commit 6edc31ce2e
2 changed files with 1 additions and 2 deletions
@@ -7,7 +7,7 @@ if __name__ == "__main__":
    
    dataset = load_dataset(
        "opencsg/chinese-cosmopedia", 
-        data_files=[f"0000{i}.parquet" for i in range(5)]
+        data_files={"train": [f"data/0000{i}.parquet" for i in range(5)]}
    )
    
    process_dataset(