ViperEkura
6ae1828449
- 删除 Registry 中未使用的 category/priority 字段，_entries 简化为直接存储类引用 - 修正 __init_subclass__ 避免叶子类（AutoRegressiveLM 等）创建空注册表 - 删除 5 个工厂的薄 create() 覆写，统一使用 BaseFactory.create(name, *args, **kwargs) - 删除 3 处零调用的 available_types/available_strategies 别名死代码 - 删除零调用的 BaseModelConfig.to_file 死代码 - 将 BaseConfig.from_json/to_json 重命名为 from_file/to_file，消除与子类重复 - 移除两个 inference builder 中总是被覆写的 prompt_tokens=0
39 lines
1.0 KiB
Python
39 lines
1.0 KiB
Python
"""CLI: JSONL → tokenized .h5/.bin via config-driven Pipeline."""
|
|
|
|
import argparse
|
|
|
|
from astrai.config.preprocess_config import PipelineConfig
|
|
from astrai.preprocessing.pipeline import Pipeline
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Raw JSONL → tokenized .h5/.bin via config-driven Pipeline"
|
|
)
|
|
parser.add_argument(
|
|
"inputs", nargs="+", metavar="JSONL", help="One or more JSONL files"
|
|
)
|
|
parser.add_argument("--output_dir", "-o", required=True, help="Output directory")
|
|
parser.add_argument(
|
|
"--config", "-c", required=True, help="Path to pipeline config JSON"
|
|
)
|
|
parser.add_argument(
|
|
"--tokenizer_path",
|
|
default="params",
|
|
help="Path to tokenizer directory (default: params)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
config = PipelineConfig.from_file(args.config)
|
|
|
|
Pipeline(
|
|
config=config,
|
|
input_paths=args.inputs,
|
|
output_dir=args.output_dir,
|
|
tokenizer_path=args.tokenizer_path,
|
|
).run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|