ViperEkura
a74e5b91a3
- Store gains fetch_record/num_records alongside stream fetch/__len__ - save_bin/load_bin support per-record offsets via record_keys param - H5Store/MmapStore/JsonlStore all support dual stream+record access - DPODataset/GRPODataset use fetch_record, no cross-record concat - dpo_collate_fn + collate_fn wired through TrainConfig - fixes attention context leakage in DPO from windowed concatenation
46 lines
906 B
Python
46 lines
906 B
Python
"""Serialization utilities for models and datasets.
|
|
|
|
This package re-exports checkpoint helpers and dataset storage helpers so
|
|
that existing imports from ``astrai.serialization`` continue to work.
|
|
"""
|
|
|
|
from astrai.serialization.checkpoint import (
|
|
Checkpoint,
|
|
load_json,
|
|
load_model_config,
|
|
load_model_weights,
|
|
load_safetensors,
|
|
load_state_dict,
|
|
load_torch,
|
|
save_json,
|
|
save_model,
|
|
save_safetensors,
|
|
save_torch,
|
|
)
|
|
from astrai.serialization.dataset import (
|
|
load_bin,
|
|
load_bin_offsets,
|
|
load_h5,
|
|
save_bin,
|
|
save_h5,
|
|
)
|
|
|
|
__all__ = [
|
|
"Checkpoint",
|
|
"load_json",
|
|
"load_model_config",
|
|
"load_model_weights",
|
|
"load_safetensors",
|
|
"load_state_dict",
|
|
"load_torch",
|
|
"save_json",
|
|
"save_model",
|
|
"save_safetensors",
|
|
"save_torch",
|
|
"load_bin",
|
|
"load_bin_offsets",
|
|
"load_h5",
|
|
"save_bin",
|
|
"save_h5",
|
|
]
|