ViperEkura
d7cd69fef5
- StreamingSeqDataset yields windows sequentially through each shard - Shard-level shuffle, distributed and multi-worker shard partitioning - __len__ returns total window count for scheduler total_steps - Better OS page-cache locality than random-access map-style datasets
40 lines
736 B
Python
40 lines
736 B
Python
from astrai.dataset.dataset import (
|
|
BaseDataset,
|
|
DatasetFactory,
|
|
dpo_collate_fn,
|
|
grpo_collate_fn,
|
|
)
|
|
from astrai.dataset.sampler import RDSampler
|
|
from astrai.dataset.storage import (
|
|
JsonlStore,
|
|
MmapStore,
|
|
Recordable,
|
|
Store,
|
|
StoreFactory,
|
|
Streamable,
|
|
detect_format,
|
|
)
|
|
from astrai.dataset.streaming import StreamingSeqDataset
|
|
from astrai.serialization import (
|
|
load_bin,
|
|
save_bin,
|
|
)
|
|
|
|
__all__ = [
|
|
"BaseDataset",
|
|
"DatasetFactory",
|
|
"dpo_collate_fn",
|
|
"grpo_collate_fn",
|
|
"Store",
|
|
"Streamable",
|
|
"Recordable",
|
|
"StoreFactory",
|
|
"MmapStore",
|
|
"JsonlStore",
|
|
"detect_format",
|
|
"save_bin",
|
|
"load_bin",
|
|
"RDSampler",
|
|
"StreamingSeqDataset",
|
|
]
|