refactor: remove H5 storage backend in favor of mmap bin
- Remove H5Store, H5Writer, save_h5/load_h5 and h5py dependency - MmapStore (bin) is the sole pre-tokenized storage backend - Move setup_logging after imports to fix E402 in __init__.py - Clean up unused imports across test files - Move inline test imports to file top
This commit is contained in:
+24
-24
@@ -4,30 +4,6 @@ __author__ = "ViperEkura"
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
def setup_logging(level: str = "INFO"):
|
||||
"""Attach a handler to the ``astrai`` logger (only, not root).
|
||||
|
||||
Call once per process, e.g. at the top of CLI scripts.
|
||||
Set ``ASTR_LOG_LEVEL`` to override the default ``INFO``.
|
||||
"""
|
||||
_logger = logging.getLogger("astrai")
|
||||
if _logger.handlers:
|
||||
return
|
||||
_level = getattr(
|
||||
logging, os.environ.get("ASTR_LOG_LEVEL", level).upper(), logging.INFO
|
||||
)
|
||||
_logger.setLevel(_level)
|
||||
_handler = logging.StreamHandler()
|
||||
_handler.setFormatter(
|
||||
logging.Formatter(
|
||||
"%(asctime)s | %(levelname)-7s | %(name)s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
)
|
||||
_logger.addHandler(_handler)
|
||||
|
||||
|
||||
from astrai.config import (
|
||||
AutoRegressiveLMConfig,
|
||||
BaseModelConfig,
|
||||
@@ -80,6 +56,30 @@ from astrai.trainer import (
|
||||
Trainer,
|
||||
)
|
||||
|
||||
|
||||
def setup_logging(level: str = "INFO"):
|
||||
"""Attach a handler to the ``astrai`` logger (only, not root).
|
||||
|
||||
Call once per process, e.g. at the top of CLI scripts.
|
||||
Set ``ASTR_LOG_LEVEL`` to override the default ``INFO``.
|
||||
"""
|
||||
_logger = logging.getLogger("astrai")
|
||||
if _logger.handlers:
|
||||
return
|
||||
_level = getattr(
|
||||
logging, os.environ.get("ASTR_LOG_LEVEL", level).upper(), logging.INFO
|
||||
)
|
||||
_logger.setLevel(_level)
|
||||
_handler = logging.StreamHandler()
|
||||
_handler.setFormatter(
|
||||
logging.Formatter(
|
||||
"%(asctime)s | %(levelname)-7s | %(name)s | %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
)
|
||||
_logger.addHandler(_handler)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AutoRegressiveLM",
|
||||
"AutoRegressiveLMConfig",
|
||||
|
||||
Reference in New Issue
Block a user