ViperEkura
a74e5b91a3
- Store gains fetch_record/num_records alongside stream fetch/__len__ - save_bin/load_bin support per-record offsets via record_keys param - H5Store/MmapStore/JsonlStore all support dual stream+record access - DPODataset/GRPODataset use fetch_record, no cross-record concat - dpo_collate_fn + collate_fn wired through TrainConfig - fixes attention context leakage in DPO from windowed concatenation
54 lines
1.5 KiB
Python
54 lines
1.5 KiB
Python
from astrai.dataset import RDSampler
|
|
|
|
|
|
def test_random_sampler_consistency(random_dataset):
|
|
"""Test RandomSampler produces consistent results with same seed"""
|
|
dataset = random_dataset
|
|
|
|
# Create two samplers with same seed
|
|
sampler1 = RDSampler(dataset, seed=42)
|
|
sampler2 = RDSampler(dataset, seed=42)
|
|
|
|
indices1 = list(iter(sampler1))
|
|
indices2 = list(iter(sampler2))
|
|
|
|
assert indices1 == indices2
|
|
|
|
|
|
def test_random_sampler_different_seeds(random_dataset):
|
|
"""Test RandomSampler produces different results with different seeds"""
|
|
dataset = random_dataset
|
|
|
|
# Create two samplers with different seeds
|
|
sampler1 = RDSampler(dataset, seed=42)
|
|
sampler2 = RDSampler(dataset, seed=123)
|
|
|
|
indices1 = list(iter(sampler1))
|
|
indices2 = list(iter(sampler2))
|
|
|
|
# Very high probability they should be different
|
|
assert indices1 != indices2
|
|
|
|
|
|
def test_sampler_across_epochs(random_dataset):
|
|
"""Test sampler behavior across multiple epochs"""
|
|
dataset = random_dataset
|
|
n = len(dataset)
|
|
|
|
sampler = RDSampler(dataset, seed=42)
|
|
|
|
# Get indices for first epoch
|
|
epoch1_indices = list(iter(sampler))
|
|
assert len(epoch1_indices) == n
|
|
|
|
# Get indices for second epoch
|
|
epoch2_indices = list(iter(sampler))
|
|
assert len(epoch2_indices) == n
|
|
|
|
# Check that epochs have different order (should be random)
|
|
assert epoch1_indices != epoch2_indices
|
|
|
|
# Check that all indices are present in each epoch
|
|
assert set(epoch1_indices) == set(range(n))
|
|
assert set(epoch2_indices) == set(range(n))
|