refactor: 抽取 BaseStorage 存储抽象,支持 JSON 原始文本数据加载

- 新增 astrai/dataset/storage.py:BaseStorage/H5Storage/JSONStorage + Fetchers + 序列化函数
- BaseDataset.load() 接入存储抽象,自动检测 HDF5/JSON 格式
- JSON 支持原始文本 + tokenizer callable 加载时 tokenize
- 新增 BaseDataset.count / keys 属性进行长度观测
- serialization.py 精简为只保留 Checkpoint 类
- 函数放前、类放后,删除分隔注释
This commit is contained in:
2026-05-12 11:17:24 +08:00
parent 38e18fdfd3
commit 5889179c54
5 changed files with 539 additions and 176 deletions
+24 -6
View File
@@ -1,19 +1,37 @@
from astrai.dataset.dataset import (
BaseDataset,
BaseSegmentFetcher,
DatasetFactory,
MultiSegmentFetcher,
)
from astrai.dataset.sampler import ResumableDistributedSampler
from astrai.dataset.storage import (
BaseSegmentFetcher,
BaseStorage,
H5Storage,
JSONStorage,
MultiSegmentFetcher,
available_storage_types,
create_storage,
detect_format,
load_h5,
load_json,
save_h5,
save_json,
)
__all__ = [
# Base classes
"BaseDataset",
# Factory
"DatasetFactory",
# Fetchers
"BaseSegmentFetcher",
"MultiSegmentFetcher",
# Sampler
"BaseStorage",
"H5Storage",
"JSONStorage",
"create_storage",
"detect_format",
"available_storage_types",
"save_h5",
"load_h5",
"save_json",
"load_json",
"ResumableDistributedSampler",
]