ViperEkura
b33250dc28
- Extract tokenization/mask/position logic from JsonlStore into TokenizeTransform - JsonlStore now pure reader: reads JSON records, delegates to transform - Store no longer imports tokenizer or preprocessing components - Replace per_record param with segments_are_records class attribute - Store subclasses declare segment semantics as format-level property
39 lines
961 B
Python
39 lines
961 B
Python
from astrai.preprocessing.builder import (
|
|
BaseMaskBuilder,
|
|
MaskBuilderFactory,
|
|
MultiOutputMaskBuilder,
|
|
SectionedMaskBuilder,
|
|
SingleOutputMaskBuilder,
|
|
)
|
|
from astrai.preprocessing.packing import (
|
|
PackingStrategy,
|
|
PackingStrategyFactory,
|
|
)
|
|
from astrai.preprocessing.pipeline import Pipeline, filter_by_length
|
|
from astrai.preprocessing.position_id import (
|
|
PositionIdStrategy,
|
|
PositionIdStrategyFactory,
|
|
)
|
|
from astrai.preprocessing.transform import TokenizeTransform
|
|
from astrai.preprocessing.writer import (
|
|
StoreWriter,
|
|
StoreWriterFactory,
|
|
)
|
|
|
|
__all__ = [
|
|
"BaseMaskBuilder",
|
|
"MaskBuilderFactory",
|
|
"MultiOutputMaskBuilder",
|
|
"PackingStrategy",
|
|
"PackingStrategyFactory",
|
|
"Pipeline",
|
|
"PositionIdStrategy",
|
|
"PositionIdStrategyFactory",
|
|
"SectionedMaskBuilder",
|
|
"SingleOutputMaskBuilder",
|
|
"StoreWriter",
|
|
"StoreWriterFactory",
|
|
"TokenizeTransform",
|
|
"filter_by_length",
|
|
]
|