refactor : Pipeline 去除去重，ids 重命名为 sequence，泛型透传

- 移除 Pipeline 内置去重逻辑及 dedup_signature 工具函数
- 删除 ProcessingConfig.deduplicate 字段
- builder 返回 'sequence' 替代 'ids'，与 dataset 层统一
- pipeline 纯透传，泛型处理任意 key 补齐默认值

astrai/config/preprocess_config.py

@@ -16,7 +16,6 @@ class ProcessingConfig(BaseConfig):
     max_seq_len: int = 2048
     min_chars: int = 50
     max_chars: int = 2_000_000
-    deduplicate: bool = True
     max_items: Optional[int] = None
 
 

astrai/preprocessing/__init__.py

@@ -3,13 +3,12 @@ from astrai.preprocessing.builder import (
     MaskBuilderFactory,
     SectionedMaskBuilder,
 )
-from astrai.preprocessing.pipeline import Pipeline, dedup_signature, filter_by_length
+from astrai.preprocessing.pipeline import Pipeline, filter_by_length
 
 __all__ = [
     "BaseMaskBuilder",
     "MaskBuilderFactory",
     "SectionedMaskBuilder",
     "Pipeline",
-    "dedup_signature",
     "filter_by_length",
 ]

astrai/preprocessing/builder.py

@@ -151,7 +151,7 @@ class SectionedMaskBuilder(BaseMaskBuilder):
             return None
 
         result: dict = {
-            "ids": all_ids,
+            "sequence": all_ids,
             "domain": _extract_domain(item, config.output.domain_key),
         }
         if not all(m == 1 for m in loss_mask):

astrai/preprocessing/pipeline.py

@@ -1,10 +1,9 @@
 """Config-driven JSONL preprocessing pipeline.
 
 Composes a :class:`BaseMaskBuilder` (selected by ``input.type``) with
-deduplication, sharding, and flush to ``.h5`` / ``.bin`` storage.
+sharding and flush to ``.h5`` / ``.bin`` storage.
 """
 
-import hashlib
 import json
 import os
 from collections import defaultdict
@@ -36,11 +35,6 @@ def filter_by_length(text: str, min_len: int = 50, max_len: int = 2_000_000) ->
     return min_len <= len(text) <= max_len
 
 
-def dedup_signature(item: dict) -> str:
-    raw = json.dumps(item, sort_keys=True, ensure_ascii=False)
-    return hashlib.md5(raw[:200].encode()).hexdigest()
-
-
 class Pipeline:
     """Tokenization pipeline driven by a declarative :class:`PipelineConfig`.
 
@@ -70,8 +64,6 @@ class Pipeline:
 
     def run(self):
         self._tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_path)
-
-        seen: set = set()
         domains: dict = defaultdict(lambda: defaultdict(list))
         total_tokens = 0
         shard_idx: dict[str, int] = defaultdict(int)
@@ -85,24 +77,23 @@ class Pipeline:
             if pp.max_items and count >= pp.max_items:
                 break
 
-            if pp.deduplicate:
-                sig = dedup_signature(item)
-                if sig in seen:
-                    continue
-                seen.add(sig)
-
             result = self.transform(item)
             if result is None:
                 continue
 
-            ids = result["ids"]
+            ids = result.pop("sequence")
             if not ids:
                 continue
 
-            domain = result.get("domain", "__default__")
-            domains[domain]["sequence"].append(ids)
-            if "loss_mask" in result:
-                domains[domain]["loss_mask"].append(result["loss_mask"])
+            domain = result.pop("domain", "__default__")
+            result["sequence"] = ids
+
+            bucket = domains[domain]
+            for key in list(bucket.keys()):
+                if key not in result:
+                    bucket[key].append([1] * len(ids))
+            for key, val in result.items():
+                bucket[key].append(val)
 
             count += 1
             total_tokens += len(ids)

astrai/tokenize/chat_template.py

@@ -29,7 +29,7 @@ class ChatTemplate:
         self.description = description
         self.default_variables = default_variables or {}
         self.special_tokens = special_tokens or {}
-        self._compiled : Template = Template(template_str)
+        self._compiled: Template = Template(template_str)
 
     @classmethod
     def from_string(