Files
AstrAI/astrai/dataset/__init__.py
T
ViperEkura 7c99da155c refactor: 删除数据流中的 JSONStore
- 移除 JSONStore 及相关函数,训练框架不再依赖 tokenizer
- Store 层只保留 H5Store 和 MmapStore 两种后端
2026-05-28 15:54:26 +08:00

32 lines
547 B
Python

from astrai.dataset.dataset import (
BaseDataset,
DatasetFactory,
)
from astrai.dataset.sampler import ResumableDistributedSampler
from astrai.dataset.storage import (
H5Store,
MmapStore,
Store,
StoreFactory,
detect_format,
load_bin,
load_h5,
save_bin,
save_h5,
)
__all__ = [
"BaseDataset",
"DatasetFactory",
"Store",
"StoreFactory",
"H5Store",
"MmapStore",
"detect_format",
"save_h5",
"load_h5",
"save_bin",
"load_bin",
"ResumableDistributedSampler",
]