refactor: split Store into StreamStore and RecordStore

- StreamStore: fetch(begin, end, key) for stream access (SEQ/SFT)
- RecordStore: mixin with fetch_record(i, key) for record access
- H5Store/MmapStore/JsonlStore now dual-inherit both (C3 MRO)
- JsonlStore supports lazy mode via processor= (no TokenizeTransform)
- RecordDataset base class holds processor, DPO/GRPO simplified
- dpo_tokenize pure function for on-the-fly JSONL tokenisation
- DatasetFactory builds processor for jsonl+record datasets
- train.py passes tokenizer_path=param_path uniformly
- progress: len(dataset) returns sample count (stream=windows, record=records)
- json no longer auto-detected as jsonl format
This commit is contained in:
2026-07-18 23:04:31 +08:00
parent b33250dc28
commit b133fc9c07
5 changed files with 461 additions and 277 deletions
+8 -6
View File
@@ -460,12 +460,13 @@ def test_dataset_load_explicit_storage_type(base_test_env):
def _write_json_dataset(test_dir, tokenizer_path, records, config_overrides=None):
"""Write JSON (not JSONL) dataset — array of objects."""
"""Write JSONL dataset — one JSON object per line."""
data_dir = os.path.join(test_dir, "json_data")
os.makedirs(data_dir, exist_ok=True)
with open(os.path.join(data_dir, "data.json"), "w", encoding="utf-8") as f:
json.dump(records, f, ensure_ascii=False)
with open(os.path.join(data_dir, "data.jsonl"), "w", encoding="utf-8") as f:
for rec in records:
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
config = {
"tokenizer_path": tokenizer_path,
@@ -544,7 +545,7 @@ def test_json_store_no_tokenizer_path(base_test_env):
# Save tokenizer files directly in the dataset directory
tokenizer.save_pretrained(data_dir)
# Write .json data
# Write .jsonl data
records = [
{
"messages": [
@@ -553,8 +554,9 @@ def test_json_store_no_tokenizer_path(base_test_env):
]
}
]
with open(os.path.join(data_dir, "data.json"), "w", encoding="utf-8") as f:
json.dump(records, f, ensure_ascii=False)
with open(os.path.join(data_dir, "data.jsonl"), "w", encoding="utf-8") as f:
for rec in records:
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
# dataset_config.json WITHOUT tokenizer_path
config = {