ViperEkura
31c22dc043
- Extract shared core (mask building, primary-id extraction, tensorisation, position-id generation) to astrai/preprocessing/core.py; Pipeline and TokenizeTransform both consume it, eliminating ~60% duplicated logic - Promote BFD _plan to module-level plan_bfd(lengths, max_len) returning pure index bins; BFDPacking.apply and evaluate_ifd._pack_bins both call it, removing the second BFD implementation - Split Pipeline._flush (49 lines) into _inject_doc_reset_position_ids + _inject_continuous_position_ids + _to_tensors; split Pipeline.run by delegating record iteration to core.iter_raw_records - Remove dead no-op pop/塞回 in Pipeline.run (L110-111)
41 lines
991 B
Python
41 lines
991 B
Python
from astrai.preprocessing.builder import (
|
|
BaseMaskBuilder,
|
|
MaskBuilderFactory,
|
|
MultiOutputMaskBuilder,
|
|
SectionedMaskBuilder,
|
|
SingleOutputMaskBuilder,
|
|
)
|
|
from astrai.preprocessing.packing import (
|
|
PackingStrategy,
|
|
PackingStrategyFactory,
|
|
plan_bfd,
|
|
)
|
|
from astrai.preprocessing.pipeline import Pipeline, filter_by_length
|
|
from astrai.preprocessing.position_id import (
|
|
PositionIdStrategy,
|
|
PositionIdStrategyFactory,
|
|
)
|
|
from astrai.preprocessing.transform import TokenizeTransform
|
|
from astrai.preprocessing.writer import (
|
|
StoreWriter,
|
|
StoreWriterFactory,
|
|
)
|
|
|
|
__all__ = [
|
|
"BaseMaskBuilder",
|
|
"MaskBuilderFactory",
|
|
"MultiOutputMaskBuilder",
|
|
"PackingStrategy",
|
|
"PackingStrategyFactory",
|
|
"Pipeline",
|
|
"PositionIdStrategy",
|
|
"PositionIdStrategyFactory",
|
|
"SectionedMaskBuilder",
|
|
"SingleOutputMaskBuilder",
|
|
"StoreWriter",
|
|
"StoreWriterFactory",
|
|
"TokenizeTransform",
|
|
"filter_by_length",
|
|
"plan_bfd",
|
|
]
|