feat: add JSONL dataset store with on-the-fly tokenization

- Add JsonlStore registered under "jsonl" in astrai/dataset/storage.py
- Reuse PipelineConfig schema for JSONL dataset configuration
- Update detect_format to recognize JSONL directories and files
- Move save_h5/load_h5/save_bin/load_bin to astrai/serialization
- Split astrai/serialization.py into checkpoint/dataset submodules
- Add tests for JSONL detection, seq/SFT stores, and config roundtrip
This commit is contained in:
2026-07-04 15:42:33 +08:00
parent 1adca39cd8
commit 8999ca89b8
8 changed files with 386 additions and 73 deletions
+4
View File
@@ -5,10 +5,13 @@ from astrai.dataset.dataset import (
from astrai.dataset.sampler import ResumableDistributedSampler
from astrai.dataset.storage import (
H5Store,
JsonlStore,
MmapStore,
Store,
StoreFactory,
detect_format,
)
from astrai.serialization import (
load_bin,
load_h5,
save_bin,
@@ -22,6 +25,7 @@ __all__ = [
"StoreFactory",
"H5Store",
"MmapStore",
"JsonlStore",
"detect_format",
"save_h5",
"load_h5",