fix: remove blocking cleanup from streaming generator
- stream finally froze main thread on cache.task_free - scheduler handles cleanup in next loop iteration instead
This commit is contained in:
@@ -204,7 +204,6 @@ class InferenceEngine:
|
|||||||
|
|
||||||
def gen():
|
def gen():
|
||||||
nonlocal remaining
|
nonlocal remaining
|
||||||
try:
|
|
||||||
while remaining > 0:
|
while remaining > 0:
|
||||||
items = result.pop_all()
|
items = result.pop_all()
|
||||||
for idx, token in items:
|
for idx, token in items:
|
||||||
@@ -216,9 +215,6 @@ class InferenceEngine:
|
|||||||
yield (idx, token) if is_batch else token
|
yield (idx, token) if is_batch else token
|
||||||
if remaining > 0:
|
if remaining > 0:
|
||||||
result.wait(timeout=0.05)
|
result.wait(timeout=0.05)
|
||||||
finally:
|
|
||||||
for tid in task_ids:
|
|
||||||
self.scheduler.remove_task(tid)
|
|
||||||
|
|
||||||
return gen()
|
return gen()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user