refactor: eliminate test duplication via shared helpers
- Add tests/helpers.py with shared config, dataset, tokenizer, executor, and assertion helpers - Replace 15 copies of device one-liner with session-scoped fixture - Collapse 5 near-identical Dataset subclasses into RandomTokenDataset - Remove duplicate _make_config/_make_model/_make_frozen and FakeTokenizer/FakeExecutor definitions - Make test_callbacks and test_early_stopping use existing train_config_factory - Replace 6 duplicate meta.json read blocks with load_shard_meta - Fix mkdtemp leaks in test_lora.py with TemporaryDirectory
This commit is contained in:
@@ -12,19 +12,7 @@ from astrai.model.transformer import AutoRegressiveLM
|
||||
from astrai.trainer.rollout import BaseRewardModel
|
||||
from astrai.trainer.schedule import SchedulerFactory
|
||||
from astrai.trainer.trainer import Trainer
|
||||
|
||||
_CHAT_TEMPLATE = (
|
||||
"{% for message in messages %}"
|
||||
"{% if message['role'] == 'system' %}"
|
||||
"SYSTEM: {{ message['content'] }}\n"
|
||||
"{% elif message['role'] == 'user' %}"
|
||||
"USER: {{ message['content'] }}\n"
|
||||
"{% elif message['role'] == 'assistant' %}"
|
||||
"ASSISTANT: {{ message['content'] }}\n"
|
||||
"{% endif %}"
|
||||
"{% endfor %}"
|
||||
"{% if add_generation_prompt %}ASSISTANT: {% endif %}"
|
||||
)
|
||||
from tests.helpers import CHAT_TEMPLATE
|
||||
|
||||
|
||||
class InstructionDataset(Dataset):
|
||||
@@ -97,7 +85,7 @@ def test_online_dpo_end_to_end(base_test_env):
|
||||
|
||||
# Equip tokenizer with a chat template so RolloutGenerator can
|
||||
# render instruction/input via apply_chat_template.
|
||||
tokenizer.set_chat_template(_CHAT_TEMPLATE)
|
||||
tokenizer.set_chat_template(CHAT_TEMPLATE)
|
||||
tokenizer.save_pretrained(test_dir)
|
||||
|
||||
model_fn = partial(_model_fn, model_config)
|
||||
|
||||
Reference in New Issue
Block a user