feat: support SFT directly from JSONL without dataset_config.json

- JsonlStore falls back to built-in messages config when no config file found and tokenizer_path is provided
- DatasetFactory.load forwards tokenizer_path to store for SFT/SEQ+jsonl
- assistant turns train, other roles masked, position_ids doc_reset
This commit is contained in:
2026-07-20 17:25:09 +08:00
parent 37a3036934
commit e0f102c4d9
3 changed files with 137 additions and 15 deletions
+9 -1
View File
@@ -346,7 +346,15 @@ class DatasetFactory(BaseFactory["BaseDataset"]):
if processor is not None:
store.load(load_path, processor=processor, **kwargs)
else:
store.load(load_path, **kwargs)
load_kwargs = dict(kwargs)
if (
tokenizer_path is not None
and storage_type == "jsonl"
and train_type in ("seq", "sft")
and "tokenizer_path" not in load_kwargs
):
load_kwargs["tokenizer_path"] = tokenizer_path
store.load(load_path, **load_kwargs)
return cls.create(train_type, store=store)