refactor : 统一 SectionedMaskBuilder,支持可配置 dtype

- 三合一 MaskBuilder,移除 chat/instruction/text,统一为 sections 配置
- OutputConfig 增加 dtype 字段 (per-key,默认 int32)
- 移除 from __future__ import annotations
- 测试适配新配置格式
This commit is contained in:
2026-05-31 14:24:10 +08:00
parent 2a65c3314c
commit dbe5891201
5 changed files with 330 additions and 206 deletions
+3 -8
View File
@@ -1,20 +1,14 @@
"""Pipeline configuration for JSONL preprocessing."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Dict, Optional
from typing import Dict, List, Optional
from astrai.config.base import BaseConfig
@dataclass
class InputConfig(BaseConfig):
type: str = "chat"
messages_key: str = "messages"
prompt_key: str = "prompt"
response_key: str = "response"
text_key: str = "text"
sections: Optional[List[Dict]] = None
@dataclass
@@ -31,6 +25,7 @@ class OutputConfig(BaseConfig):
domain_key: Optional[str] = None
storage_format: str = "bin"
max_tokens_per_shard: int = 100_000_000
dtype: Dict[str, str] = field(default_factory=dict)
@dataclass