ViperEkura
598e1ce4ae
- 将 pipeline/packing.py 拆分为 packing/ 子包 (base/stream/binpack) - 新增 BfdPacker(默认)/FfDPacker/GreedyPacker，移除 StreamingPacker - 超长序列直接截断至 pack_size - group_size 语义改为"每 N 个 chunk 合并为一块"，默认 1000 - 新增 AutoTokenizer.token_to_id()，修复 ChatML 中 hacky 的 nl_id 获取 - pad_value 默认改为 2(pad_token_id)，position_ids pad=0, loss_mask pad=False - 新增 position_ids 打包后归零一致性测试 - scripts/cache_h5.py 新增 --pack-algo 参数
123 lines
3.7 KiB
Python
123 lines
3.7 KiB
Python
"""Tests for strategy module."""
|
||
|
||
import pytest
|
||
from pipeline.strategies import (
|
||
PromptStrategy,
|
||
ChatMLStrategy,
|
||
AlpacaStrategy,
|
||
StrategyFactory,
|
||
)
|
||
|
||
|
||
class DummyTokenizer:
|
||
def encode(self, text: str, add_special_tokens: bool = False):
|
||
return [ord(c) for c in text]
|
||
|
||
def token_to_id(self, token: str):
|
||
return ord(token)
|
||
|
||
|
||
class DummyStrategy(PromptStrategy):
|
||
def __init__(self, tokenizer):
|
||
super().__init__(tokenizer)
|
||
|
||
@property
|
||
def name(self) -> str:
|
||
return "dummy"
|
||
|
||
def assemble_prompt(self, query_tokens):
|
||
prefix = self._encode_format("Q: ")
|
||
return prefix + query_tokens
|
||
|
||
def assemble_response(self, response_tokens):
|
||
suffix = self._encode_format("<｜end▁of▁sentence｜>")
|
||
return response_tokens + suffix
|
||
|
||
|
||
def _decode(tokens):
|
||
return "".join(chr(t) for t in tokens)
|
||
|
||
|
||
class TestChatMLStrategy:
|
||
def test_name(self):
|
||
assert ChatMLStrategy(DummyTokenizer()).name == "chatml"
|
||
|
||
def test_assemble_prompt(self):
|
||
tk = DummyTokenizer()
|
||
strategy = ChatMLStrategy(tk)
|
||
query_tokens = tk.encode("hello")
|
||
prompt = strategy.assemble_prompt(query_tokens)
|
||
text = _decode(prompt)
|
||
assert "<｜im▁start｜>user" in text
|
||
assert "hello" in text
|
||
assert "<｜im▁start｜>assistant" in text
|
||
|
||
def test_assemble_response(self):
|
||
tk = DummyTokenizer()
|
||
strategy = ChatMLStrategy(tk)
|
||
response_tokens = tk.encode("world")
|
||
response = strategy.assemble_response(response_tokens)
|
||
text = _decode(response)
|
||
assert "world" in text
|
||
assert "<｜im▁end｜>" in text
|
||
|
||
def test_prompt_ends_with_assistant_start(self):
|
||
tk = DummyTokenizer()
|
||
strategy = ChatMLStrategy(tk)
|
||
prompt = strategy.assemble_prompt(tk.encode("hi"))
|
||
# prompt 末尾应该是 assistant_start 的 token ids
|
||
assert (
|
||
prompt[-len(strategy._assistant_start_ids) :]
|
||
== strategy._assistant_start_ids
|
||
)
|
||
|
||
|
||
class TestAlpacaStrategy:
|
||
def test_name(self):
|
||
assert AlpacaStrategy(DummyTokenizer()).name == "alpaca"
|
||
|
||
def test_assemble_prompt(self):
|
||
tk = DummyTokenizer()
|
||
strategy = AlpacaStrategy(tk)
|
||
query_tokens = tk.encode("hello")
|
||
prompt = strategy.assemble_prompt(query_tokens)
|
||
text = _decode(prompt)
|
||
assert "### Instruction:" in text
|
||
assert "hello" in text
|
||
assert "### Response:" in text
|
||
|
||
def test_assemble_response(self):
|
||
tk = DummyTokenizer()
|
||
strategy = AlpacaStrategy(tk)
|
||
response_tokens = tk.encode("world")
|
||
response = strategy.assemble_response(response_tokens)
|
||
text = _decode(response)
|
||
assert "world" in text
|
||
assert "<｜end▁of▁sentence｜>" in text
|
||
|
||
|
||
class TestStrategyFactory:
|
||
def test_create_chatml(self):
|
||
tk = DummyTokenizer()
|
||
assert isinstance(StrategyFactory.create("chatml", tk), ChatMLStrategy)
|
||
|
||
def test_create_alpaca(self):
|
||
tk = DummyTokenizer()
|
||
assert isinstance(StrategyFactory.create("alpaca", tk), AlpacaStrategy)
|
||
|
||
def test_create_invalid_raises_error(self):
|
||
with pytest.raises(ValueError, match="Unknown strategy"):
|
||
StrategyFactory.create("invalid_strategy", DummyTokenizer())
|
||
|
||
def test_register_and_create(self):
|
||
StrategyFactory.register("dummy")(DummyStrategy)
|
||
tk = DummyTokenizer()
|
||
strategy = StrategyFactory.create("dummy", tk)
|
||
assert isinstance(strategy, DummyStrategy)
|
||
assert strategy.name == "dummy"
|
||
|
||
def test_available_strategies(self):
|
||
strategies = StrategyFactory.available_types()
|
||
assert "chatml" in strategies
|
||
assert "alpaca" in strategies
|