feat: add JSONL dataset store with on-the-fly tokenization
- Add JsonlStore registered under "jsonl" in astrai/dataset/storage.py - Reuse PipelineConfig schema for JSONL dataset configuration - Update detect_format to recognize JSONL directories and files - Move save_h5/load_h5/save_bin/load_bin to astrai/serialization - Split astrai/serialization.py into checkpoint/dataset submodules - Add tests for JSONL detection, seq/SFT stores, and config roundtrip
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
"""Serialization utilities for models and datasets.
|
||||
|
||||
This package re-exports checkpoint helpers and dataset storage helpers so
|
||||
that existing imports from ``astrai.serialization`` continue to work.
|
||||
"""
|
||||
|
||||
from astrai.serialization.checkpoint import (
|
||||
Checkpoint,
|
||||
load_json,
|
||||
load_model_config,
|
||||
load_model_weights,
|
||||
load_safetensors,
|
||||
load_state_dict,
|
||||
load_torch,
|
||||
save_json,
|
||||
save_model,
|
||||
save_safetensors,
|
||||
save_torch,
|
||||
)
|
||||
from astrai.serialization.dataset import (
|
||||
load_bin,
|
||||
load_h5,
|
||||
save_bin,
|
||||
save_h5,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Checkpoint",
|
||||
"load_json",
|
||||
"load_model_config",
|
||||
"load_model_weights",
|
||||
"load_safetensors",
|
||||
"load_state_dict",
|
||||
"load_torch",
|
||||
"save_json",
|
||||
"save_model",
|
||||
"save_safetensors",
|
||||
"save_torch",
|
||||
"load_bin",
|
||||
"load_h5",
|
||||
"save_bin",
|
||||
"save_h5",
|
||||
]
|
||||
Reference in New Issue
Block a user