From 0690f66bedabe590e296859e1ba14b03ad2cc7f0 Mon Sep 17 00:00:00 2001
From: ViperEkura <3081035982@qq.com>
Date: Mon, 30 Jun 2025 21:44:04 +0800
Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AF=B9jsonl=20=E6=96=87?=
 =?UTF-8?q?=E4=BB=B6=E7=9A=84=E5=A4=84=E7=90=86bug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 to_ids.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/to_ids.py b/to_ids.py
index 6cdea2d..f445eb0 100644
--- a/to_ids.py
+++ b/to_ids.py
@@ -1,6 +1,7 @@
 from tokenizer import BpeTokenizer
 from tqdm import tqdm
 from typing import List
+import json
 import pickle as pkl
 import torch
 import os
@@ -12,7 +13,8 @@ def convert_to_ids(tokenizer: BpeTokenizer, file_path, out_file_path):
     arrows = []
     with open(file_path, "r") as f:
         for line in f:
-            ids = tokenizer.encode(line)
+            line = json.loads(line)
+            ids = tokenizer.encode(line["text"])
             arrows.append(ids)
     
     with open(out_file_path, "w") as f: