ViperEkura
841a582b28
- Extract SingleOutputMaskBuilder for SFT and pretrain configs - Extract MultiOutputMaskBuilder for DPO and GRPO configs - Keep SectionedMaskBuilder as backward-compatible facade - Register "single" and "multi" names in MaskBuilderFactory - Add parity and rejection tests for concrete builders
37 lines
875 B
Python
37 lines
875 B
Python
from astrai.preprocessing.builder import (
|
|
BaseMaskBuilder,
|
|
MaskBuilderFactory,
|
|
MultiOutputMaskBuilder,
|
|
SectionedMaskBuilder,
|
|
SingleOutputMaskBuilder,
|
|
)
|
|
from astrai.preprocessing.packing import (
|
|
PackingStrategy,
|
|
PackingStrategyFactory,
|
|
)
|
|
from astrai.preprocessing.pipeline import Pipeline, filter_by_length
|
|
from astrai.preprocessing.position_id import (
|
|
PositionIdStrategy,
|
|
PositionIdStrategyFactory,
|
|
)
|
|
from astrai.preprocessing.writer import (
|
|
StoreWriter,
|
|
StoreWriterFactory,
|
|
)
|
|
|
|
__all__ = [
|
|
"BaseMaskBuilder",
|
|
"MaskBuilderFactory",
|
|
"MultiOutputMaskBuilder",
|
|
"PackingStrategy",
|
|
"PackingStrategyFactory",
|
|
"Pipeline",
|
|
"PositionIdStrategy",
|
|
"PositionIdStrategyFactory",
|
|
"SectionedMaskBuilder",
|
|
"SingleOutputMaskBuilder",
|
|
"StoreWriter",
|
|
"StoreWriterFactory",
|
|
"filter_by_length",
|
|
]
|