refactor: deduplicate low-risk code paths
This commit is contained in:
@@ -383,10 +383,10 @@ class DatasetFactory(BaseFactory["BaseDataset"]):
|
||||
transform = _build_jsonl_transform(load_path, tokenizer_path)
|
||||
if transform is None:
|
||||
raise FileNotFoundError(
|
||||
f"JSONL dataset config not found. Expected "
|
||||
f"dataset_config.json alongside *.jsonl files, pass "
|
||||
f"tokenizer_path= for the built-in messages config, or "
|
||||
f"use processor= for lazy on-the-fly tokenisation."
|
||||
"JSONL dataset config not found. Expected "
|
||||
"dataset_config.json alongside *.jsonl files, pass "
|
||||
"tokenizer_path= for the built-in messages config, or "
|
||||
"use processor= for lazy on-the-fly tokenisation."
|
||||
)
|
||||
store.load(load_path, transform=transform, **kwargs)
|
||||
else:
|
||||
|
||||
@@ -217,7 +217,7 @@ class Store(ABC):
|
||||
"""
|
||||
if self._window_size <= 0:
|
||||
raise RuntimeError("sample_window() requires window_size > 0 (stream mode)")
|
||||
if self._window_size <= 0 or self._length <= self._window_size:
|
||||
if self._length <= self._window_size:
|
||||
raise IndexError(
|
||||
f"Data too short for window: token_count={self._length}, "
|
||||
f"window_size={self._window_size}"
|
||||
|
||||
Reference in New Issue
Block a user