refactor : BaseConfig 提供 from_json/to_json，嵌套 config 自动反序列化

- from_json/to_json 上提至 BaseConfig，所有子类自动继承
- _coerce 新增 dict 到 BaseConfig 子类的递归反序列化，消除子类 from_dict 重载
- PipelineConfig 等子类仅声明字段，零样板代码
- 测试 tokenizer 改为自包含 BPE（含 chat template），不依赖 params/ 目录
- 特殊 token 改用 ASCII 字符，兼容所有平台

astrai/config/base.py

@@ -1,6 +1,7 @@
 import json
 from dataclasses import MISSING, dataclass, fields
-from typing import Any, Dict, Optional, Self, get_type_hints
+from pathlib import Path
+from typing import Any, Dict, Optional, Self, Union, get_type_hints
 
 
 @dataclass
@@ -83,4 +84,15 @@ class BaseConfig:
             return value
         if isinstance(value, target_type):
             return value
+        if isinstance(value, dict) and issubclass(target_type, BaseConfig):
+            return target_type.from_dict(value)
         raise TypeError
+
+    @classmethod
+    def from_json(cls, path: Union[str, Path]) -> Self:
+        with open(path, "r", encoding="utf-8") as f:
+            return cls.from_dict(json.load(f))
+
+    def to_json(self, path: Union[str, Path]):
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)

astrai/config/preprocess_config.py

@@ -2,13 +2,14 @@
 
 from __future__ import annotations
 
-import json
 from dataclasses import dataclass, field
 from typing import Dict, Optional
 
+from astrai.config.base import BaseConfig
+
 
 @dataclass
-class InputConfig:
+class InputConfig(BaseConfig):
     type: str = "chat"
     messages_key: str = "messages"
     prompt_key: str = "prompt"
@@ -17,7 +18,7 @@ class InputConfig:
 
 
 @dataclass
-class ProcessingConfig:
+class ProcessingConfig(BaseConfig):
     max_seq_len: int = 2048
     min_chars: int = 50
     max_chars: int = 2_000_000
@@ -26,63 +27,17 @@ class ProcessingConfig:
 
 
 @dataclass
-class OutputConfig:
+class OutputConfig(BaseConfig):
     domain_key: Optional[str] = None
     storage_format: str = "bin"
     max_tokens_per_shard: int = 100_000_000
 
 
 @dataclass
-class PipelineConfig:
+class PipelineConfig(BaseConfig):
     version: int = 1
     input: InputConfig = field(default_factory=InputConfig)
     mask: Dict[str, str] = field(default_factory=dict)
     mask_default: str = "mask"
     preprocessing: ProcessingConfig = field(default_factory=ProcessingConfig)
     output: OutputConfig = field(default_factory=OutputConfig)
-
-    def to_dict(self) -> dict:
-        return {
-            "version": self.version,
-            "input": {
-                "type": self.input.type,
-                "messages_key": self.input.messages_key,
-                "prompt_key": self.input.prompt_key,
-                "response_key": self.input.response_key,
-                "text_key": self.input.text_key,
-            },
-            "mask": self.mask,
-            "mask_default": self.mask_default,
-            "preprocessing": {
-                "max_seq_len": self.preprocessing.max_seq_len,
-                "min_chars": self.preprocessing.min_chars,
-                "max_chars": self.preprocessing.max_chars,
-                "deduplicate": self.preprocessing.deduplicate,
-                "max_items": self.preprocessing.max_items,
-            },
-            "output": {
-                "domain_key": self.output.domain_key,
-                "storage_format": self.output.storage_format,
-                "max_tokens_per_shard": self.output.max_tokens_per_shard,
-            },
-        }
-
-    @classmethod
-    def from_dict(cls, data: dict) -> PipelineConfig:
-        return PipelineConfig(
-            version=data.get("version", 1),
-            input=InputConfig(**data.get("input", {})),
-            mask=data.get("mask", {}),
-            mask_default=data.get("mask_default", "mask"),
-            preprocessing=ProcessingConfig(**data.get("preprocessing", {})),
-            output=OutputConfig(**data.get("output", {})),
-        )
-
-    @classmethod
-    def from_json(cls, path: str) -> PipelineConfig:
-        with open(path, "r", encoding="utf-8") as f:
-            return cls.from_dict(json.load(f))
-
-    def to_json(self, path: str):
-        with open(path, "w", encoding="utf-8") as f:
-            json.dump(self.to_dict(), f, indent=2, ensure_ascii=False)