refactor: remove dead code and deduplicate scheduler setup

2026-08-19 14:58:37 +08:00
parent 00c2c80c8f
commit f252af495c
8 changed files with 24 additions and 101 deletions
@@ -492,9 +492,6 @@ class DPODataset(BaseDataset):

    required_keys = ["chosen", "rejected", "chosen_mask", "rejected_mask"]

-    def make_processor(self, tokenizer, max_len: int):
-        return partial(dpo_processor, tokenizer=tokenizer, max_len=max_len)
-
    def __getitem__(self, index: int) -> Dict[str, Tensor]:
        return {
            "chosen": self.store.fetch_record(index, "chosen").to(dtype=torch.long),