39 lines
880 B
TOML
39 lines
880 B
TOML
[build-system]
|
|
requires = ["setuptools>=64", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "datapipline"
|
|
version = "0.1.0"
|
|
description = "A dataset processing toolkit for language model training"
|
|
readme = "README.md"
|
|
requires-python = ">=3.10"
|
|
dependencies = [
|
|
"tokenizers>=0.13.0",
|
|
"datasets>=2.10.0",
|
|
"numpy>=1.21.0",
|
|
"tqdm>=4.64.0",
|
|
"torch>=2.1.0",
|
|
"h5py>=3.7.0",
|
|
]
|
|
keywords = ["nlp", "datasets", "language-models", "machine-learning"]
|
|
[project.urls]
|
|
Homepage = "https://github.com/khaosz/khaosz_dataset"
|
|
|
|
[tool.setuptools.packages.find]
|
|
where = ["."]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|
|
python_files = ["test_*.py"]
|
|
python_classes = ["Test*"]
|
|
python_functions = ["test_*"]
|
|
addopts = "-v --tb=short"
|
|
filterwarnings = [
|
|
"ignore::DeprecationWarning",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=7.0.0",
|
|
] |