ViperEkura
115192c67c
- Remove H5Store, H5Writer, save_h5/load_h5 and h5py dependency - MmapStore (bin) is the sole pre-tokenized storage backend - Move setup_logging after imports to fix E402 in __init__.py - Clean up unused imports across test files - Move inline test imports to file top
57 lines
1.5 KiB
Python
57 lines
1.5 KiB
Python
"""Storage writer strategies for pipeline output.
|
|
|
|
The :class:`StoreWriter` abstraction decouples the pipeline from the
|
|
concrete storage format (bin). The pipeline builds a ``{key:
|
|
List[Tensor]}`` dict and delegates the write to the writer selected
|
|
by ``output.storage_format``.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import shutil
|
|
from abc import ABC, abstractmethod
|
|
from typing import Dict, List
|
|
|
|
import torch
|
|
|
|
from astrai.factory import BaseFactory
|
|
from astrai.serialization import save_bin
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class StoreWriter(ABC):
|
|
"""Write pre-tokenized tensors to disk in a format-specific way."""
|
|
|
|
@abstractmethod
|
|
def save(
|
|
self,
|
|
output_dir: str,
|
|
domain: str,
|
|
shard_idx: int,
|
|
tensors: Dict[str, List[torch.Tensor]],
|
|
) -> None: ...
|
|
|
|
|
|
class StoreWriterFactory(BaseFactory["StoreWriter"]):
|
|
pass
|
|
|
|
|
|
@StoreWriterFactory.register("bin")
|
|
class BinWriter(StoreWriter):
|
|
def save(self, output_dir, domain, shard_idx, tensors):
|
|
shard_path = os.path.join(output_dir, domain, f"shard_{shard_idx:04d}")
|
|
try:
|
|
save_bin(shard_path, tensors)
|
|
except Exception:
|
|
if os.path.exists(shard_path):
|
|
shutil.rmtree(shard_path, ignore_errors=True)
|
|
logger.error(
|
|
"Failed to write shard %s/%s_%04d, cleaned up partial output",
|
|
domain,
|
|
"shard",
|
|
shard_idx,
|
|
exc_info=True,
|
|
)
|
|
raise
|