ViperEkura
a1ea26d367
- process_list_field returns List[List[int]] preserving per-response boundaries - GRPODataset rewritten to record-level __getitem__ (no windowing/stride) - grpo_collate_fn pads variable-length responses into [B, G, R] tensors - JsonlStore detects nested List[List[int]] and stores List[Tensor] per record - Store._normalize skips nested-list keys from cumsum bookkeeping - Pipeline._flush handles nested lists without cross-record flattening - Export grpo_collate_fn from astrai.dataset - 6 new GRPO tests + 2 updated builder tests, 114 total pass
38 lines
662 B
Python
38 lines
662 B
Python
from astrai.dataset.dataset import (
|
|
BaseDataset,
|
|
DatasetFactory,
|
|
grpo_collate_fn,
|
|
)
|
|
from astrai.dataset.sampler import ResumableDistributedSampler
|
|
from astrai.dataset.storage import (
|
|
H5Store,
|
|
JsonlStore,
|
|
MmapStore,
|
|
Store,
|
|
StoreFactory,
|
|
detect_format,
|
|
)
|
|
from astrai.serialization import (
|
|
load_bin,
|
|
load_h5,
|
|
save_bin,
|
|
save_h5,
|
|
)
|
|
|
|
__all__ = [
|
|
"BaseDataset",
|
|
"DatasetFactory",
|
|
"grpo_collate_fn",
|
|
"Store",
|
|
"StoreFactory",
|
|
"H5Store",
|
|
"MmapStore",
|
|
"JsonlStore",
|
|
"detect_format",
|
|
"save_h5",
|
|
"load_h5",
|
|
"save_bin",
|
|
"load_bin",
|
|
"ResumableDistributedSampler",
|
|
]
|