ViperEkura
b133fc9c07
- StreamStore: fetch(begin, end, key) for stream access (SEQ/SFT) - RecordStore: mixin with fetch_record(i, key) for record access - H5Store/MmapStore/JsonlStore now dual-inherit both (C3 MRO) - JsonlStore supports lazy mode via processor= (no TokenizeTransform) - RecordDataset base class holds processor, DPO/GRPO simplified - dpo_tokenize pure function for on-the-fly JSONL tokenisation - DatasetFactory builds processor for jsonl+record datasets - train.py passes tokenizer_path=param_path uniformly - progress: len(dataset) returns sample count (stream=windows, record=records) - json no longer auto-detected as jsonl format
44 lines
740 B
Python
44 lines
740 B
Python
from astrai.dataset.dataset import (
|
|
BaseDataset,
|
|
DatasetFactory,
|
|
dpo_collate_fn,
|
|
grpo_collate_fn,
|
|
)
|
|
from astrai.dataset.sampler import RDSampler
|
|
from astrai.dataset.storage import (
|
|
H5Store,
|
|
JsonlStore,
|
|
MmapStore,
|
|
RecordStore,
|
|
Store,
|
|
StoreFactory,
|
|
StreamStore,
|
|
detect_format,
|
|
)
|
|
from astrai.serialization import (
|
|
load_bin,
|
|
load_h5,
|
|
save_bin,
|
|
save_h5,
|
|
)
|
|
|
|
__all__ = [
|
|
"BaseDataset",
|
|
"DatasetFactory",
|
|
"dpo_collate_fn",
|
|
"grpo_collate_fn",
|
|
"Store",
|
|
"StreamStore",
|
|
"RecordStore",
|
|
"StoreFactory",
|
|
"H5Store",
|
|
"MmapStore",
|
|
"JsonlStore",
|
|
"detect_format",
|
|
"save_h5",
|
|
"load_h5",
|
|
"save_bin",
|
|
"load_bin",
|
|
"RDSampler",
|
|
]
|