refactor : Storage 层重构为 Store,移除 Fetcher 中间层,支持多段数据与显式长度
- 合并 BaseStorage + MultiSegmentFetcher + BaseSegmentFetcher 三层为 Store ABC - Store._data 直接持有 Dict[str, List[Tensor]],不做强制拼接避免 OOM - _fetch_key 统一用 bisect 跨段切片,单段多段同一路径 - _length 显式存储(min total across keys),__len__ 返回 O(1) - MmapStore/H5Store/JSONStore 统一走 _normalize() 注册分段并预计算累积长度 - 所有 I/O 函数 (save_h5/load_h5/json_to_bin 等) 保持不变
This commit is contained in:
+10
-14
@@ -4,13 +4,11 @@ from astrai.dataset.dataset import (
|
||||
)
|
||||
from astrai.dataset.sampler import ResumableDistributedSampler
|
||||
from astrai.dataset.storage import (
|
||||
BaseSegmentFetcher,
|
||||
BaseStorage,
|
||||
H5Storage,
|
||||
JSONStorage,
|
||||
MmapStorage,
|
||||
MultiSegmentFetcher,
|
||||
StorageFactory,
|
||||
H5Store,
|
||||
JSONStore,
|
||||
MmapStore,
|
||||
Store,
|
||||
StoreFactory,
|
||||
detect_format,
|
||||
json_to_bin,
|
||||
load_bin,
|
||||
@@ -24,13 +22,11 @@ from astrai.dataset.storage import (
|
||||
__all__ = [
|
||||
"BaseDataset",
|
||||
"DatasetFactory",
|
||||
"BaseSegmentFetcher",
|
||||
"MultiSegmentFetcher",
|
||||
"BaseStorage",
|
||||
"H5Storage",
|
||||
"JSONStorage",
|
||||
"MmapStorage",
|
||||
"StorageFactory",
|
||||
"Store",
|
||||
"StoreFactory",
|
||||
"H5Store",
|
||||
"JSONStore",
|
||||
"MmapStore",
|
||||
"detect_format",
|
||||
"save_h5",
|
||||
"load_h5",
|
||||
|
||||
Reference in New Issue
Block a user