fix: cache_h5 移除重复 --batch-size 参数,默认输出格式改为 bin
This commit is contained in:
+2
-8
@@ -66,12 +66,6 @@ def main():
|
||||
default=1_000,
|
||||
help="Merge every N packed chunks into one tensor, <=0 to disable (default: 1000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=256,
|
||||
help="Records tokenized per batch (default: 256)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
default="INFO",
|
||||
@@ -87,9 +81,9 @@ def main():
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--output-format",
|
||||
default="h5",
|
||||
default="bin",
|
||||
choices=["h5", "bin"],
|
||||
help="Output format: h5 or bin (default: h5)",
|
||||
help="Output format: h5 or bin (default: bin)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user