refactor : pipeline 策略化拆分,消除 _flush if/else

- PackingStrategy / PositionIdStrategy / StoreWriter 独立文件 + Factory
- Pipeline._flush 零 if/else,纯编排
- SectionRenderer 从 SectionedMaskBuilder 分离
- OutputConfig.position_ids_mode 默认改为 ""none""
This commit is contained in:
2026-06-06 00:45:33 +08:00
parent 3057741de9
commit 31bc7f5c2a
7 changed files with 424 additions and 306 deletions
+2 -2
View File
@@ -87,7 +87,7 @@ class OutputConfig(BaseConfig):
position_ids_mode : Optional[str]
How to compute position_ids in packed sequences.
- ``None`` / ``"none"``: do not generate (backward compatible).
- ``"none"``: do not generate (default).
- ``"doc_reset"``: reset to 0 at each document boundary.
- ``"continuous"``: sequential 0, 1, 2, ... (pretrain, single doc).
"""
@@ -96,7 +96,7 @@ class OutputConfig(BaseConfig):
storage_format: str = "bin"
max_tokens_per_shard: int = 100_000_000
dtype: Dict[str, str] = field(default_factory=dict)
position_ids_mode: Optional[str] = None
position_ids_mode: str = "none"
@dataclass