diff --git a/astrai/inference/engine.py b/astrai/inference/engine.py index e7a2d48..77cd1d5 100644 --- a/astrai/inference/engine.py +++ b/astrai/inference/engine.py @@ -148,7 +148,7 @@ class InferenceEngine: tokenizer: AutoTokenizer, max_batch_size: int = 1, max_seq_len: Optional[int] = None, - max_prompt_len: int = 512, + max_prompt_len: int = 2048, cache_capacity: int = 1000, ): """Initializes the engine and starts the scheduler background thread. diff --git a/astrai/inference/scheduler.py b/astrai/inference/scheduler.py index c9b690e..2c387a5 100644 --- a/astrai/inference/scheduler.py +++ b/astrai/inference/scheduler.py @@ -480,7 +480,7 @@ class InferenceScheduler: prompt_ids = self.tokenizer.encode(prompt) if len(prompt_ids) > self.max_prompt_len: - prompt_ids = prompt_ids[: self.max_prompt_len] + prompt_ids = prompt_ids[-self.max_prompt_len :] task = Task( task_id=task_id,