ViperEkura/DataPipeline
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

修复对jsonl 文件的处理bug

Browse Source
This commit is contained in:
ViperEkura
2025-06-30 21:44:04 +08:00
parent 34dc9632d8
commit 0690f66bed
1 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
to_ids.py
+3 -1
View File
@@ -1,6 +1,7 @@
from tokenizer import BpeTokenizer from tokenizer import BpeTokenizer
from tqdm import tqdm from tqdm import tqdm
from typing import List from typing import List
import json
import pickle as pkl import pickle as pkl
import torch import torch
import os import os
@@ -12,7 +13,8 @@ def convert_to_ids(tokenizer: BpeTokenizer, file_path, out_file_path):
arrows = [] arrows = []
with open(file_path, "r") as f: with open(file_path, "r") as f:
for line in f: for line in f:
ids = tokenizer.encode(line) line = json.loads(line)
ids = tokenizer.encode(line["text"])
arrows.append(ids) arrows.append(ids)
with open(out_file_path, "w") as f: with open(out_file_path, "w") as f: