refactor : 统一 SectionedMaskBuilder,支持可配置 dtype
- 三合一 MaskBuilder,移除 chat/instruction/text,统一为 sections 配置 - OutputConfig 增加 dtype 字段 (per-key,默认 int32) - 移除 from __future__ import annotations - 测试适配新配置格式
This commit is contained in:
@@ -1,20 +1,14 @@
|
||||
"""Pipeline configuration for JSONL preprocessing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, Optional
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from astrai.config.base import BaseConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class InputConfig(BaseConfig):
|
||||
type: str = "chat"
|
||||
messages_key: str = "messages"
|
||||
prompt_key: str = "prompt"
|
||||
response_key: str = "response"
|
||||
text_key: str = "text"
|
||||
sections: Optional[List[Dict]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -31,6 +25,7 @@ class OutputConfig(BaseConfig):
|
||||
domain_key: Optional[str] = None
|
||||
storage_format: str = "bin"
|
||||
max_tokens_per_shard: int = 100_000_000
|
||||
dtype: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in New Issue
Block a user