feat: add JSONL dataset store with on-the-fly tokenization
- Add JsonlStore registered under "jsonl" in astrai/dataset/storage.py - Reuse PipelineConfig schema for JSONL dataset configuration - Update detect_format to recognize JSONL directories and files - Move save_h5/load_h5/save_bin/load_bin to astrai/serialization - Split astrai/serialization.py into checkpoint/dataset submodules - Add tests for JSONL detection, seq/SFT stores, and config roundtrip
This commit is contained in:
@@ -5,10 +5,13 @@ from astrai.dataset.dataset import (
|
||||
from astrai.dataset.sampler import ResumableDistributedSampler
|
||||
from astrai.dataset.storage import (
|
||||
H5Store,
|
||||
JsonlStore,
|
||||
MmapStore,
|
||||
Store,
|
||||
StoreFactory,
|
||||
detect_format,
|
||||
)
|
||||
from astrai.serialization import (
|
||||
load_bin,
|
||||
load_h5,
|
||||
save_bin,
|
||||
@@ -22,6 +25,7 @@ __all__ = [
|
||||
"StoreFactory",
|
||||
"H5Store",
|
||||
"MmapStore",
|
||||
"JsonlStore",
|
||||
"detect_format",
|
||||
"save_h5",
|
||||
"load_h5",
|
||||
|
||||
Reference in New Issue
Block a user