refactor: remove H5 storage backend in favor of mmap bin

- Remove H5Store, H5Writer, save_h5/load_h5 and h5py dependency
- MmapStore (bin) is the sole pre-tokenized storage backend
- Move setup_logging after imports to fix E402 in __init__.py
- Clean up unused imports across test files
- Move inline test imports to file top
This commit is contained in:
2026-07-29 12:50:27 +08:00
parent c2b04d8458
commit 115192c67c
17 changed files with 91 additions and 209 deletions
-6
View File
@@ -6,7 +6,6 @@ from astrai.dataset.dataset import (
)
from astrai.dataset.sampler import RDSampler
from astrai.dataset.storage import (
H5Store,
JsonlStore,
MmapStore,
Recordable,
@@ -17,9 +16,7 @@ from astrai.dataset.storage import (
)
from astrai.serialization import (
load_bin,
load_h5,
save_bin,
save_h5,
)
__all__ = [
@@ -31,12 +28,9 @@ __all__ = [
"Streamable",
"Recordable",
"StoreFactory",
"H5Store",
"MmapStore",
"JsonlStore",
"detect_format",
"save_h5",
"load_h5",
"save_bin",
"load_bin",
"RDSampler",