refactor: assemble inference engines through a shared composition root
- add build_engine() to astrai.inference.engine as the single load-place-wire path for InferenceEngine, accepting a checkpoint path or live model/tokenizer plus passthrough engine kwargs - migrate the server lifespan, generate CLI, humaneval/ifeval evals, and all three demos to build_engine; app._create_engine collapses into a direct call - export build_engine from astrai and astrai.inference - parameterize the autoregressive demo with --prompt one-shot continuation plus model path and sampling knobs, exiting cleanly on !exit or EOF - cover the composition root with unit tests for live-object assembly, kwargs passthrough, and argument validation
This commit is contained in:
@@ -1,11 +1,7 @@
|
||||
from argparse import ArgumentParser
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
|
||||
from astrai import InferenceEngine
|
||||
from astrai.model import AutoModel
|
||||
from astrai.tokenize import AutoTokenizer
|
||||
from astrai import build_engine
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -68,10 +64,8 @@ def chat():
|
||||
args = parse_args()
|
||||
model_path = args.model_path
|
||||
|
||||
model = AutoModel.from_pretrained(model_path)
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path)
|
||||
model.to(device="cuda", dtype=torch.bfloat16)
|
||||
engine = InferenceEngine(model=model, tokenizer=tokenizer)
|
||||
engine = build_engine(model_path)
|
||||
tokenizer = engine.tokenizer
|
||||
|
||||
while True:
|
||||
query = input(">> ")
|
||||
|
||||
Reference in New Issue
Block a user