refactor: deduplicate and restructure test suite

- extract preprocessing config factories into tests/data/factories.py
- keep conftest.py fixtures-only; stop importing builders from it
- promote temp_dir fixture to root conftest for cross-directory reuse
- unify duplicate BPE tokenizer builders into build_test_tokenizer
- merge grpo/dpo online e2e tests into one parametrized integration test
- extract engine mock factory and shared model batch builders
- drop local tempfile usage in favor of shared fixtures

No behavior change: 519 tests pass.

tests/data/conftest.py

@@ -1,21 +1,15 @@
 import json
 import os
-import tempfile
 
 import pytest
-from tokenizers import Tokenizer, models, pre_tokenizers, trainers
 
-from astrai.config.preprocess_config import (
-    InputConfig,
-    PipelineConfig,
-    ProcessingConfig,
-)
 from astrai.preprocessing.builder import (
     MultiOutputMaskBuilder,
     SectionedMaskBuilder,
     SingleOutputMaskBuilder,
 )
-from astrai.tokenize import AutoTokenizer
+from tests.data.factories import make_grpo_config
+from tests.helpers import build_test_tokenizer
 
 _SPECIAL_TOKENS_CONFIG = {
     "bos_token": "<|begin_of_sentence|>",
@@ -41,55 +35,42 @@ _CHAT_TEMPLATE = (
     "{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
 )
 
-_CHAT_SECTIONS = [{"field": "messages", "action": "$role", "template": True}]
 
-_INSTRUCTION_SECTIONS = [
-    {"field": "prompt", "action": "mask", "add_special_tokens": True},
-    {"field": "response", "action": "train"},
+_CHAT_TOKENIZER_DATA = [
+    "hello world",
+    "Hi there!",
+    "You are helpful.",
+    "What is 2+2?",
+    "Tell me a story about dragons and knights.",
+    "Sure, here is a tale.",
+    "Translate to French: Hello",
+    "Bonjour",
+    "Artificial Intelligence is a field of computer science.",
+    "system",
+    "user",
+    "assistant",
+    "<|im_start|>",
+    "<|im_end|>",
+    *[chr(i) for i in range(32, 127)],
 ]
 
-_TEXT_SECTIONS = [{"field": "text", "action": "train"}]
-
-_GRPO_RESPONSE_SECTIONS = [{"field": "responses", "action": "train"}]
+_CHAT_TOKENIZER_MAP = {
+    "bos_token": "<|begin_of_sentence|>",
+    "eos_token": "<|end_of_sentence|>",
+    "pad_token": "<|_pad_|>",
+    "unk_token": "<|_unk_|>",
+}
 
 
 def _build_chat_tokenizer():
-    tok = Tokenizer(models.BPE())
-    tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
-    tr = trainers.BpeTrainer(
+    return build_test_tokenizer(
         vocab_size=512,
-        min_frequency=1,
         special_tokens=_SPECIAL_TOKENS,
+        special_token_map=_CHAT_TOKENIZER_MAP,
+        add_prefix_space=False,
+        train_data=_CHAT_TOKENIZER_DATA,
+        chat_template=_CHAT_TEMPLATE,
     )
-    train_data = [
-        "hello world",
-        "Hi there!",
-        "You are helpful.",
-        "What is 2+2?",
-        "Tell me a story about dragons and knights.",
-        "Sure, here is a tale.",
-        "Translate to French: Hello",
-        "Bonjour",
-        "Artificial Intelligence is a field of computer science.",
-        "system",
-        "user",
-        "assistant",
-        "<|im_start|>",
-        "<|im_end|>",
-        *[chr(i) for i in range(32, 127)],
-    ]
-    tok.train_from_iterator(train_data, tr)
-
-    auto_tok = AutoTokenizer()
-    auto_tok._tokenizer = tok
-    auto_tok._special_token_map = {
-        "bos_token": "<|begin_of_sentence|>",
-        "eos_token": "<|end_of_sentence|>",
-        "pad_token": "<|_pad_|>",
-        "unk_token": "<|_unk_|>",
-    }
-    auto_tok.set_chat_template(_CHAT_TEMPLATE)
-    return auto_tok
 
 
 @pytest.fixture(scope="session")
@@ -97,116 +78,11 @@ def chat_tokenizer():
     return _build_chat_tokenizer()
 
 
-@pytest.fixture
-def temp_dir():
-    d = tempfile.mkdtemp()
-    yield d
-    import shutil
-
-    shutil.rmtree(d, ignore_errors=True)
-
-
-def make_chat_config():
-    return PipelineConfig(
-        input=InputConfig(sections=_CHAT_SECTIONS),
-        mask={"system": "mask", "user": "mask", "assistant": "train"},
-        mask_default="mask",
-        preprocessing=ProcessingConfig(max_seq_len=2048),
-    )
-
-
-def make_instruction_config():
-    return PipelineConfig(
-        input=InputConfig(sections=_INSTRUCTION_SECTIONS),
-        mask={"prompt": "mask", "response": "train"},
-        mask_default="mask",
-        preprocessing=ProcessingConfig(max_seq_len=2048),
-    )
-
-
-def make_text_config():
-    return PipelineConfig(
-        input=InputConfig(sections=_TEXT_SECTIONS),
-        preprocessing=ProcessingConfig(
-            max_seq_len=2048, min_chars=1, max_chars=2_000_000
-        ),
-    )
-
-
-def make_dpo_chat_config():
-    return PipelineConfig(
-        input=InputConfig(
-            sources={
-                "chosen": {
-                    "sections": [
-                        {"field": "chosen", "action": "$role", "template": True}
-                    ]
-                },
-                "rejected": {
-                    "sections": [
-                        {"field": "rejected", "action": "$role", "template": True}
-                    ]
-                },
-            }
-        ),
-        mask={"user": "mask", "assistant": "train"},
-        mask_default="mask",
-        preprocessing=ProcessingConfig(max_seq_len=2048),
-    )
-
-
-def make_grpo_config():
-    return PipelineConfig(
-        input=InputConfig(
-            sources={
-                "prompts": {
-                    "sections": [
-                        {"field": "prompt", "action": "mask", "template": True}
-                    ]
-                },
-                "responses": {
-                    "sections": _GRPO_RESPONSE_SECTIONS,
-                    "list_field": True,
-                    "mask_key": "masks",
-                },
-                "rewards": {
-                    "sections": [{"field": "rewards", "action": "value"}],
-                },
-            }
-        ),
-        mask={"user": "mask", "assistant": "train"},
-        mask_default="mask",
-        preprocessing=ProcessingConfig(max_seq_len=2048),
-    )
-
-
-def make_grpo_no_template_config():
-    return PipelineConfig(
-        input=InputConfig(
-            sources={
-                "prompts": {
-                    "sections": [
-                        {
-                            "field": "prompt",
-                            "action": "mask",
-                            "add_special_tokens": True,
-                        }
-                    ]
-                },
-                "responses": {
-                    "sections": _GRPO_RESPONSE_SECTIONS,
-                    "list_field": True,
-                    "mask_key": "masks",
-                },
-                "rewards": {
-                    "sections": [{"field": "rewards", "action": "value"}],
-                },
-            }
-        ),
-        mask={"user": "mask", "assistant": "train"},
-        mask_default="mask",
-        preprocessing=ProcessingConfig(max_seq_len=2048),
-    )
+def _write_tokenizer_dir(dir_path, tokenizer, tokenizer_config):
+    """Persist a tokenizer plus ``tokenizer_config.json`` into *dir_path*."""
+    tokenizer._tokenizer.save(os.path.join(dir_path, "tokenizer.json"))
+    with open(os.path.join(dir_path, "tokenizer_config.json"), "w") as f:
+        json.dump(tokenizer_config, f)
 
 
 @pytest.fixture
@@ -228,11 +104,11 @@ def multi_builder():
 def tokenizer_dir(temp_dir, test_tokenizer):
     d = os.path.join(temp_dir, "tok")
     os.makedirs(d, exist_ok=True)
-    test_tokenizer._tokenizer.save(os.path.join(d, "tokenizer.json"))
-    with open(os.path.join(d, "tokenizer_config.json"), "w") as f:
-        json.dump(
-            {"special_tokens": {"pad_token": "<|_pad_|>", "unk_token": "<|_unk_|>"}}, f
-        )
+    _write_tokenizer_dir(
+        d,
+        test_tokenizer,
+        {"special_tokens": {"pad_token": "<|_pad_|>", "unk_token": "<|_unk_|>"}},
+    )
     return d
 
 
@@ -240,10 +116,9 @@ def tokenizer_dir(temp_dir, test_tokenizer):
 def chat_tokenizer_dir(temp_dir, chat_tokenizer):
     d = os.path.join(temp_dir, "tok")
     os.makedirs(d, exist_ok=True)
-    chat_tokenizer._tokenizer.save(os.path.join(d, "tokenizer.json"))
-    with open(os.path.join(d, "tokenizer_config.json"), "w") as f:
-        json.dump(
-            {"special_tokens": _SPECIAL_TOKENS_CONFIG, "chat_template": _CHAT_TEMPLATE},
-            f,
-        )
+    _write_tokenizer_dir(
+        d,
+        chat_tokenizer,
+        {"special_tokens": _SPECIAL_TOKENS_CONFIG, "chat_template": _CHAT_TEMPLATE},
+    )
     return d