fix: batch 推理示例添加 chat template 和 system prompt
- 新增 prompts 列表,对每个输入应用 apply_chat_template - 添加 system message 到对话模板
This commit is contained in:
parent
d8da2cf17c
commit
f0339022c1
|
|
@ -24,12 +24,23 @@ def batch_generate():
|
||||||
"请问什么是显卡",
|
"请问什么是显卡",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
prompts = [
|
||||||
|
tokenizer.apply_chat_template(
|
||||||
|
[
|
||||||
|
{"role": "system", "content": "You are a helpful assistant."},
|
||||||
|
{"role": "user", "content": q},
|
||||||
|
],
|
||||||
|
tokenize=False,
|
||||||
|
)
|
||||||
|
for q in inputs
|
||||||
|
]
|
||||||
|
|
||||||
engine = InferenceEngine(
|
engine = InferenceEngine(
|
||||||
model=model,
|
model=model,
|
||||||
tokenizer=tokenizer,
|
tokenizer=tokenizer,
|
||||||
)
|
)
|
||||||
responses = engine.generate(
|
responses = engine.generate(
|
||||||
prompt=inputs,
|
prompt=prompts,
|
||||||
stream=False,
|
stream=False,
|
||||||
max_tokens=2048,
|
max_tokens=2048,
|
||||||
temperature=0.8,
|
temperature=0.8,
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue