refactor: 删除数据流中的 JSONStore

- 移除 JSONStore 及相关函数,训练框架不再依赖 tokenizer
- Store 层只保留 H5Store 和 MmapStore 两种后端
This commit is contained in:
2026-05-28 15:54:26 +08:00
parent 629e72385b
commit 7c99da155c
4 changed files with 20 additions and 381 deletions
-8
View File
@@ -5,18 +5,14 @@ from astrai.dataset.dataset import (
from astrai.dataset.sampler import ResumableDistributedSampler
from astrai.dataset.storage import (
H5Store,
JSONStore,
MmapStore,
Store,
StoreFactory,
detect_format,
json_to_bin,
load_bin,
load_h5,
load_json,
save_bin,
save_h5,
save_json,
)
__all__ = [
@@ -25,15 +21,11 @@ __all__ = [
"Store",
"StoreFactory",
"H5Store",
"JSONStore",
"MmapStore",
"detect_format",
"save_h5",
"load_h5",
"save_json",
"load_json",
"save_bin",
"load_bin",
"json_to_bin",
"ResumableDistributedSampler",
]