92 lines
2.8 KiB
Python
92 lines
2.8 KiB
Python
import concurrent.futures
|
|
import time
|
|
|
|
import pytest
|
|
import requests
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_rate_limit_and_queue(router_manager, mock_workers):
|
|
# One fast backend
|
|
_, urls, _ = mock_workers(n=1)
|
|
rh = router_manager.start_router(
|
|
worker_urls=urls,
|
|
policy="round_robin",
|
|
extra={
|
|
"max_concurrent_requests": 2,
|
|
"queue_size": 0, # no queue -> immediate 429 when limit exceeded
|
|
},
|
|
)
|
|
|
|
def call_once(i):
|
|
try:
|
|
r = requests.post(
|
|
f"{rh.url}/v1/completions",
|
|
json={
|
|
"model": "test-model",
|
|
"prompt": f"p{i}",
|
|
"max_tokens": 1,
|
|
"stream": False,
|
|
},
|
|
timeout=3,
|
|
)
|
|
return r.status_code
|
|
except Exception:
|
|
return 599
|
|
|
|
# Fire a burst of concurrent requests
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex:
|
|
results = list(ex.map(call_once, range(16)))
|
|
|
|
# Expect some to succeed and some to be rate limited (429)
|
|
assert any(code == 200 for code in results)
|
|
assert any(code == 429 for code in results)
|
|
|
|
|
|
@pytest.mark.integration
|
|
def test_rate_limit_queue_and_timeout(router_manager, mock_workers):
|
|
# Slow backend: ~2s per request ensures queue wait > timeout
|
|
_, urls, _ = mock_workers(n=1, args=["--latency-ms", "2000"]) # 2.0s per request
|
|
|
|
# Allow 1 concurrent, queue up to 1, with 1s queue timeout
|
|
rh = router_manager.start_router(
|
|
worker_urls=urls,
|
|
policy="round_robin",
|
|
extra={
|
|
"max_concurrent_requests": 1,
|
|
"queue_size": 1,
|
|
"queue_timeout_secs": 1,
|
|
},
|
|
)
|
|
|
|
def call_once(i):
|
|
try:
|
|
r = requests.post(
|
|
f"{rh.url}/v1/completions",
|
|
json={
|
|
"model": "test-model",
|
|
"prompt": f"q{i}",
|
|
"max_tokens": 1,
|
|
"stream": False,
|
|
},
|
|
timeout=5,
|
|
)
|
|
return r.status_code
|
|
except Exception:
|
|
return 599
|
|
|
|
# Fire 4 concurrent requests: 1 runs (~2s), 1 queued (times out at 1s -> 408), 2 overflow -> 429
|
|
import concurrent.futures
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
|
|
results = list(ex.map(call_once, range(4)))
|
|
|
|
# We expect:
|
|
# - Some 200s (processed)
|
|
# - At least one 408 (queued too long and timed out)
|
|
# - Remaining non-200s are either 429 (queue overflow) or additional 408s depending on scheduling
|
|
assert any(code == 200 for code in results)
|
|
assert any(code == 408 for code in results), results
|
|
non200 = [c for c in results if c != 200]
|
|
assert len(non200) >= 2 and all(c in (408, 429) for c in non200), results
|