feat: add JSONL dataset store with on-the-fly tokenization

- Add JsonlStore registered under "jsonl" in astrai/dataset/storage.py
- Reuse PipelineConfig schema for JSONL dataset configuration
- Update detect_format to recognize JSONL directories and files
- Move save_h5/load_h5/save_bin/load_bin to astrai/serialization
- Split astrai/serialization.py into checkpoint/dataset submodules
- Add tests for JSONL detection, seq/SFT stores, and config roundtrip
This commit is contained in:
2026-07-04 15:42:33 +08:00
parent 1adca39cd8
commit 8999ca89b8
8 changed files with 386 additions and 73 deletions
+43
View File
@@ -0,0 +1,43 @@
"""Serialization utilities for models and datasets.
This package re-exports checkpoint helpers and dataset storage helpers so
that existing imports from ``astrai.serialization`` continue to work.
"""
from astrai.serialization.checkpoint import (
Checkpoint,
load_json,
load_model_config,
load_model_weights,
load_safetensors,
load_state_dict,
load_torch,
save_json,
save_model,
save_safetensors,
save_torch,
)
from astrai.serialization.dataset import (
load_bin,
load_h5,
save_bin,
save_h5,
)
__all__ = [
"Checkpoint",
"load_json",
"load_model_config",
"load_model_weights",
"load_safetensors",
"load_state_dict",
"load_torch",
"save_json",
"save_model",
"save_safetensors",
"save_torch",
"load_bin",
"load_h5",
"save_bin",
"save_h5",
]