diff --git a/chinese-c4.py b/chinese-c4.py index f0934f6..172270c 100644 --- a/chinese-c4.py +++ b/chinese-c4.py @@ -18,7 +18,7 @@ if __name__ == "__main__": end_idx = min((i + 1) * chunk_size, total_samples) chunk = train_dataset.select(range(start_idx, end_idx)) - output_path = f"{output_dir}/{output_dir}_text_chunk_{i}.jsonl" + output_path = f"{output_dir}/chinese-c4_text_chunk_{i}.jsonl" with open(output_path, "w", encoding="utf-8") as f: for example in chunk: # 每行写入一个 {"text": "xxx"} 对象 diff --git a/chinese-wiki.py b/chinese-wiki.py index 2a2e13a..815a811 100644 --- a/chinese-wiki.py +++ b/chinese-wiki.py @@ -10,7 +10,7 @@ if __name__ == "__main__": total_samples = len(train_dataset) num_chunks = (total_samples // chunk_size) + 1 script_dir = os.path.dirname(os.path.abspath(__file__)) - output_dir = os.path(script_dir, "dataset", "chinese-c4") + output_dir = os.path.join(script_dir, "dataset", "chinese-wiki") os.makedirs(output_dir, exist_ok=True) @@ -19,7 +19,7 @@ if __name__ == "__main__": end_idx = min((i + 1) * chunk_size, total_samples) chunk = train_dataset.select(range(start_idx, end_idx)) - output_path = f"{output_dir}/{output_dir}_text_chunk_{i}.jsonl" + output_path = f"{output_dir}/chinese-wiki_text_chunk_{i}.jsonl" with open(output_path, "w", encoding="utf-8") as f: for example in chunk: json_line = {"text": example["text"]} diff --git a/english-fineweb.py b/english-fineweb.py index ecf929c..871d606 100644 --- a/english-fineweb.py +++ b/english-fineweb.py @@ -33,7 +33,7 @@ if __name__ == "__main__": end_idx = min((i + 1) * chunk_size, total_samples) chunk = train_dataset.select(range(start_idx, end_idx)) - output_path = f"{output_dir}/{output_dir}text_chunk_{i}.jsonl" + output_path = f"{output_dir}/english-fineweb_text_chunk_{i}.jsonl" with open(output_path, "w", encoding="utf-8") as f: for example in chunk: json_line = {"text": comprehensive_normalization(example["text"])}