refactor: migrate scripts from argparse to click, add YAML config support

- Replace argparse with click in all scripts (train, server, generate,
  preprocess, benchmark)
- Add --config YAML support to train.py with CLI flag override
- Add --dry-run mode to validate config before training
- Add type annotations throughout benchmark.py
- Unify docstring format across all commands
- Remove redundant deps httpx, requests, pyyaml, rich from pyproject.toml
- Net -346 lines while adding YAML config support

scripts/tools/preprocess.py

@@ -1,48 +1,50 @@
 """CLI: JSONL → tokenized .h5/.bin via config-driven Pipeline."""
 
-import argparse
+import click
 
 from astrai.config.preprocess_config import PipelineConfig
 from astrai.preprocessing.pipeline import Pipeline
 
 
-def main():
-    parser = argparse.ArgumentParser(
-        description="Raw JSONL → tokenized .h5/.bin via config-driven Pipeline"
-    )
-    parser.add_argument(
-        "inputs", nargs="+", metavar="JSONL", help="One or more JSONL files"
-    )
-    parser.add_argument("--output_dir", "-o", required=True, help="Output directory")
-    parser.add_argument(
-        "--config", "-c", required=True, help="Path to pipeline config JSON"
-    )
-    parser.add_argument(
-        "--tokenizer_path",
-        default="params",
-        help="Path to tokenizer directory (default: params)",
-    )
-    parser.add_argument(
-        "--batch_size",
-        type=int,
-        default=None,
-        help="Number of records tokenized together (default: config value)",
-    )
-    args = parser.parse_args()
-
-    config = PipelineConfig.from_file(args.config)
-    if args.batch_size is not None:
-        if args.batch_size < 1:
-            parser.error("--batch_size must be at least 1")
-        config.preprocessing.batch_size = args.batch_size
+@click.command(
+    name="preprocess", help="Tokenize and pack raw JSONL data into .bin/.h5 format."
+)
+@click.argument("inputs", nargs=-1, type=click.Path(exists=True), required=True)
+@click.option(
+    "--output_dir", "-o", type=click.Path(), required=True, help="Output directory."
+)
+@click.option(
+    "--config",
+    "-c",
+    "pipeline_config",
+    type=click.Path(exists=True),
+    required=True,
+    help="Pipeline config JSON.",
+)
+@click.option(
+    "--tokenizer_path",
+    type=click.Path(exists=True),
+    default="params",
+    help="Path to tokenizer directory.",
+)
+@click.option("--batch_size", type=int, default=None, help="Records per batch.")
+def preprocess_command(inputs, output_dir, pipeline_config, tokenizer_path, batch_size):
+    """Tokenize and pack raw JSONL data into .bin/.h5 format."""
+    config = PipelineConfig.from_file(pipeline_config)
+    if batch_size is not None:
+        if batch_size < 1:
+            raise click.BadParameter("--batch_size must be at least 1")
+        config.preprocessing.batch_size = batch_size
 
+    click.echo(f"Preprocessing {len(inputs)} file(s) → {output_dir}")
     Pipeline(
         config=config,
-        input_paths=args.inputs,
-        output_dir=args.output_dir,
-        tokenizer_path=args.tokenizer_path,
+        input_paths=list(inputs),
+        output_dir=output_dir,
+        tokenizer_path=tokenizer_path,
     ).run()
+    click.echo("Done.")
 
 
 if __name__ == "__main__":
-    main()
+    preprocess_command()