refactor: simplify BaseFactory and separate ModelFactory from AutoModel
- Extract _resolve_base_type and _validate_component as module-level helpers - Replace ForwardRef._evaluate private API with eval in module namespace - Remove broad except Exception in __init_subclass__, _component_base always set - Replace direct _entries mutation in strategy.py with register() call form - Remove dead TOKENIZER_CLASSES registry from AutoTokenizer - Extract ModelFactory(BaseFactory[nn.Module]) as pure factory - AutoModel now inherits only nn.Module, no factory state - Move @AutoModel.register to @ModelFactory.register in transformer.py and encoder.py
This commit is contained in:
@@ -20,8 +20,6 @@ Messages = List[Message]
|
||||
class AutoTokenizer:
|
||||
"""Base tokenizer class with automatic loading support"""
|
||||
|
||||
TOKENIZER_CLASSES = {} # Registry for auto-loading
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
path: Optional[Union[str, Path]] = None,
|
||||
@@ -108,17 +106,6 @@ class AutoTokenizer:
|
||||
with open(save_path / "tokenizer_config.json", "w", encoding="utf-8") as f:
|
||||
json.dump(config, f, ensure_ascii=False, indent=2)
|
||||
|
||||
@classmethod
|
||||
def register_tokenizer(cls, name: str, tokenizer_class: type):
|
||||
"""
|
||||
Register a new tokenizer class.
|
||||
|
||||
Args:
|
||||
name: Name to register the tokenizer class under
|
||||
tokenizer_class: The tokenizer class to register
|
||||
"""
|
||||
cls.TOKENIZER_CLASSES[name] = tokenizer_class
|
||||
|
||||
def encode(
|
||||
self,
|
||||
tokens: Union[str, List[str]],
|
||||
|
||||
Reference in New Issue
Block a user