feat: 实现模型动态注册机制

2026-04-05 19:38:12 +08:00
parent ff43a2fab8
commit fc278d17ab
25 changed files with 686 additions and 651 deletions
@@ -90,8 +90,12 @@ from astrai.inference import InferenceEngine, GenerationRequest
 param = ModelParameter.load("your_model_dir")
 param.to(device="cuda", dtype=torch.bfloat16)

-# Create engine
-engine = InferenceEngine(param)
+# Create engine with separate model and tokenizer
+engine = InferenceEngine(
+    model=param.model,
+    tokenizer=param.tokenizer,
+    config=param.config,
+)

 # Build request
 request = GenerationRequest(