refactor: remove H5 storage backend in favor of mmap bin

- Remove H5Store, H5Writer, save_h5/load_h5 and h5py dependency
- MmapStore (bin) is the sole pre-tokenized storage backend
- Move setup_logging after imports to fix E402 in __init__.py
- Clean up unused imports across test files
- Move inline test imports to file top
This commit is contained in:
2026-07-29 12:50:27 +08:00
parent c2b04d8458
commit 115192c67c
17 changed files with 91 additions and 209 deletions
+3 -3
View File
@@ -1,4 +1,4 @@
"""CLI: JSONL → tokenized .h5/.bin via config-driven Pipeline."""
"""CLI: JSONL → tokenized .bin via config-driven Pipeline."""
import click
@@ -8,7 +8,7 @@ from astrai.preprocessing.pipeline import Pipeline
@click.command(
name="preprocess", help="Tokenize and pack raw JSONL data into .bin/.h5 format."
name="preprocess", help="Tokenize and pack raw JSONL data into .bin format."
)
@click.argument("inputs", nargs=-1, type=click.Path(exists=True), required=True)
@click.option(
@@ -30,7 +30,7 @@ from astrai.preprocessing.pipeline import Pipeline
)
@click.option("--batch_size", type=int, default=None, help="Records per batch.")
def preprocess_command(inputs, output_dir, pipeline_config, tokenizer_path, batch_size):
"""Tokenize and pack raw JSONL data into .bin/.h5 format."""
"""Tokenize and pack raw JSONL data into .bin format."""
config = PipelineConfig.from_file(pipeline_config)
if batch_size is not None:
if batch_size < 1: