refactor(dataset): 重构数据处理脚本并支持自定义数据集

2025-07-15 12:18:58 +08:00
parent 1c7e49a631
commit bea4f83842
5 changed files with 25 additions and 15 deletions
@@ -1,6 +1,5 @@
 from utils import process_dataset

-
 if __name__ == "__main__":
    process_dataset(
        dataset_name="HuggingFaceFW/fineweb",