Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/).

## [Unreleased]

### Changed

- **KV admission now needs `concurrency_limit: auto`.** Since 0.15.5 (#277) a root with no `concurrency_limit` on an SGLang engine `sieval run` launched was admitted by KV cache; it is now bounded only by its task and stage limits, as before 0.15.5 (no `KV:` line, no `kv_tokens`). Set `concurrency_limit: auto` to keep KV admission. Throughput only: identity schema and `RESULT_SCHEMA_VERSION` are unchanged, and 0.15.6 result directories resume.

## [0.15.6] - 2026-11-08

A patch release: `concurrency_limit: auto` (#278) works again. 0.15.5's
Expand Down
40 changes: 18 additions & 22 deletions docs/guide/concurrency.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,31 +16,12 @@ config = TaskRunnerConfig(concurrency_limits={"infer": 64, "postprocess": 32})
model = ChatModel("gpt-4o", concurrency_limit=32)
```

## KV Admission

An SGLang engine `sieval run` launches also admits requests by the KV cache
they may hold, unless its root model sets `concurrency_limit`.

- **Capacity.** `max_total_num_tokens` from `/get_server_info` once the
engine is ready, times its data-parallel ranks. The launch prints
`KV: <tokens> x <replicas>`. vLLM, external endpoints and an engine that
does not report it get none.
- **Size.** `n × min(context, max_tokens + 4096)`, a whole context per
sequence when `max_tokens` is unset. One `n: 8` request holds what eight
`n: 1` ones do; a 1024-token request holds 5120.
- **Order.** First come, first served, after the other limits. A request
waits before it is sent, so no client timeout runs meanwhile. One larger
than the whole pool runs alone.
- Throughput only: no stage identity changes. Request records show the
budget as `kv_tokens` under `in_flight`.

## `concurrency_limit: auto`

A root model on an SGLang engine `sieval run` launches can set
`concurrency_limit: auto`: KV admission as above, and while requests wait,
SiEval reads the engine's `/v1/loads` once a second. A waiting request is held
back while the engine reports queued requests or a KV pool at least 95% full.
An engine that does not answer leaves the KV budget in charge.
`concurrency_limit: auto` to admit requests by the KV cache they may hold.
Only `auto` does this: a number is a fixed cap, and no limit leaves the root
bounded by the task and stage limits alone.

```yaml
models:
Expand All @@ -51,6 +32,21 @@ models:
checkpoint: /models/candidate
```

- **Capacity.** `max_total_num_tokens` from `/get_server_info` once the
engine is ready, times its data-parallel ranks. The launch prints
`KV: <tokens> x <replicas>`.
- **Size.** `n × min(context, max_tokens + 4096)`, a whole context per
sequence when `max_tokens` is unset. One `n: 8` request holds what eight
`n: 1` ones do; a 1024-token request holds 5120.
- **Order.** First come, first served, after the other limits. A request
waits before it is sent, so no client timeout runs meanwhile. One larger
than the whole pool runs alone.
- **Engine load.** While requests wait, SiEval polls `/v1/loads` once a
second and holds them back while the engine reports queued requests or a
KV pool at least 95% full. No answer leaves the KV budget in charge.
- Throughput only: no stage identity changes. Request records show the
budget as `kv_tokens` under `in_flight`.

`--dry-run` accepts it before the engine exists; a model started at a barrier
reads its pool at its first request. If the engine reports no KV pool, its
requests fail naming the root. Anywhere else (external API, vLLM, a derived
Expand Down
25 changes: 21 additions & 4 deletions sieval/cli/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
read_persisted_config,
redact_secrets,
resolve_deterministic,
sets_auto_concurrency,
unwrap_proxies,
)
from sieval.cli.signals import terminate_as_interrupt
Expand Down Expand Up @@ -1506,14 +1507,20 @@ class _Engines:
deployments: dict[str, Deployment] = dataclasses.field(default_factory=dict)

async def start(
self, name: str, plan: DeploymentPlan, commands: list[BackendCommand]
self,
name: str,
plan: DeploymentPlan,
commands: list[BackendCommand],
*,
auto: bool,
) -> Deployment:
if name in self.launched:
return self.deployments[name]
self.deployments[name] = await _launch(
name,
plan,
commands,
auto=auto,
deployer=self.deployer,
launched=self.launched,
handle_dir=self.handle_dir,
Expand Down Expand Up @@ -1592,10 +1599,17 @@ async def _run_group(
await check_launch_gpu(required=True)
for root in sorted(group.own - runnable):
log_user("No work for model {} in this run; not starting it", group.names[root])
models = config.get("models") or {}

def auto(root: str) -> bool:
return sets_auto_concurrency(models.get(root) or {})

realized: dict[str, Deployment] = {}
for root in sorted(upfront):
plan, commands = group.prepared[root]
realized[root] = await engines.start(group.names[root], plan, commands)
realized[root] = await engines.start(
group.names[root], plan, commands, auto=auto(root)
)
for root in sorted(deferred):
# Up already, kept from an earlier group.
name = group.names[root]
Expand All @@ -1616,7 +1630,7 @@ async def on_barrier(stage: str) -> None:
for root in sorted(deferred):
if staged.starts(root) == stage:
plan, commands = group.prepared[root]
await engines.start(group.names[root], plan, commands)
await engines.start(group.names[root], plan, commands, auto=auto(root))
watches.start(root, engines.launched[group.names[root]])

serving = dataclasses.replace(serving, on_barrier=on_barrier)
Expand Down Expand Up @@ -1665,6 +1679,7 @@ async def _launch(
plan: DeploymentPlan,
commands: list[BackendCommand],
*,
auto: bool,
deployer: LocalDeployer,
launched: dict[str, list[InferHandle]],
handle_dir: Path,
Expand All @@ -1673,7 +1688,7 @@ async def _launch(
ready_timeout: float,
) -> Deployment:
"""Start one engine under *model_name*, wait until it is ready, and record
its handles in *launched*."""
its handles in *launched*; with *auto*, also read its KV pool."""
log_user("Starting inference for model: {}", model_name)

last_status = None
Expand Down Expand Up @@ -1740,6 +1755,8 @@ def _progress(elapsed: float, status_value: str) -> None:
env.gpu_count,
)
deployment = deployer.build_deployment(plan, new_handles)
if not auto:
return deployment
facts = await kv_facts(plan.backend, deployment.api_base)
if facts.kv_capacity_tokens is None:
return deployment
Expand Down
87 changes: 39 additions & 48 deletions sieval/cli/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -1257,34 +1257,39 @@ def _default_client_timeout(deployment: Deployment | None) -> httpx.Timeout:
)


#: A root model's ``concurrency_limit`` that adds the engine's own load to KV
#: admission (:class:`AdaptiveTokenBudget`).
#: The root ``concurrency_limit`` that turns on KV admission
#: (:class:`AdaptiveTokenBudget`); no other value does.
AUTO_CONCURRENCY = "auto"


def _kv_token_budget(
deployment: Deployment, *, adaptive: bool = False
) -> TokenBudget | None:
"""KV admission for an engine ``sieval run`` launched that reported
its pool; ``None`` otherwise. *adaptive* also reads an SGLang engine's
load, and is ``None`` on any other engine."""
return _budget_from_facts(deployment, deployment.facts, adaptive=adaptive)
def _configured_concurrency_limit(
config: Mapping[str, object], nested: Mapping[str, object]
) -> object:
"""``concurrency_limit``: the top level wins over ``args``."""
return config.get("concurrency_limit", nested.get("concurrency_limit"))


def sets_auto_concurrency(model_config: Mapping[str, object]) -> bool:
"""Whether a root model sets ``concurrency_limit: auto``."""
limit = _configured_concurrency_limit(model_config, _nested_args(model_config))
return limit == AUTO_CONCURRENCY


def _kv_token_budget(deployment: Deployment) -> TokenBudget | None:
"""KV admission from the pool a launched SGLang engine reported, else ``None``."""
return _budget_from_facts(deployment, deployment.facts)


def _pending_kv_token_budget(
deployment: Deployment, *, root: str, adaptive: bool
deployment: Deployment, *, root: str
) -> PendingTokenBudget:
"""KV admission for an SGLang engine not up yet, read at its first request.

The launch waits for readiness, so this is the same one read an engine
started up front gets. With no pool reported, ``auto`` fails naming the
root, and no limit runs without KV admission.
"""
"""KV admission for an SGLang engine not up yet, read at its first request;
fails naming *root* if the engine reports no pool."""

async def resolve() -> TokenBudget | None:
facts = await kv_facts(deployment.engine.engine_id, deployment.api_base)
budget = _budget_from_facts(deployment, facts, adaptive=adaptive)
if adaptive and budget is None:
budget = _budget_from_facts(deployment, facts)
if budget is None:
raise RuntimeError(
f"Model root '{root}': concurrency_limit '{AUTO_CONCURRENCY}' "
"needs the engine's KV pool and context window, and the SGLang "
Expand All @@ -1297,23 +1302,17 @@ async def resolve() -> TokenBudget | None:


def _budget_from_facts(
deployment: Deployment, facts: ServingFacts, *, adaptive: bool
deployment: Deployment, facts: ServingFacts
) -> TokenBudget | None:
if (
deployment.plan is None
or deployment.engine.engine_id != "sglang"
or facts.kv_capacity_tokens is None
or facts.context_tokens is None
):
return None
capacity = facts.kv_capacity_tokens * facts.replicas
if not adaptive:
return TokenBudget(
capacity, context=facts.context_tokens, headroom=PROMPT_HEADROOM
)
if deployment.engine.engine_id != "sglang":
return None
return AdaptiveTokenBudget(
capacity,
facts.kv_capacity_tokens * facts.replicas,
context=facts.context_tokens,
headroom=PROMPT_HEADROOM,
probe=SglangLoadProbe(deployment.api_base),
Expand Down Expand Up @@ -3972,9 +3971,7 @@ def _connection_options(
config, nested, subject, os.environ if environ is None else environ
)
max_retries = config.get("max_retries", nested.get("max_retries", 3))
concurrency_limit = config.get(
"concurrency_limit", nested.get("concurrency_limit")
)
concurrency_limit = _configured_concurrency_limit(config, nested)
if isinstance(max_retries, bool) or not isinstance(max_retries, int):
raise TypeError("max_retries must be an integer")
if concurrency_limit != AUTO_CONCURRENCY and (
Expand Down Expand Up @@ -4269,31 +4266,25 @@ def _create_owned_pool(
token_budgets = {}
self._root_token_budgets = token_budgets
if root_key not in token_budgets:
auto = concurrency_limit == AUTO_CONCURRENCY
budget: TokenBudget | PendingTokenBudget | None = (
_kv_token_budget(deployment, adaptive=auto)
if auto or concurrency_limit is None
else None
)
if budget is None and (auto or concurrency_limit is None):
if _launches_sglang(deployment) and self._not_up_yet(root_key):
budget = _pending_kv_token_budget(
deployment, root=root_key, adaptive=auto
budget: TokenBudget | PendingTokenBudget | None = None
if concurrency_limit == AUTO_CONCURRENCY:
budget = _kv_token_budget(deployment)
if budget is None and not _launches_sglang(deployment):
# Prelaunch refuses this first; kept as the backstop.
raise ValueError(
f"Model root '{root_key}': concurrency_limit "
f"'{AUTO_CONCURRENCY}' needs an SGLang engine sieval run "
"launched; set a number here"
)
elif auto and _launches_sglang(deployment):
if budget is None and self._not_up_yet(root_key):
budget = _pending_kv_token_budget(deployment, root=root_key)
if budget is None:
raise ValueError(
f"Model root '{root_key}': concurrency_limit "
f"'{AUTO_CONCURRENCY}' needs the KV pool and context "
"window, and the SGLang engine sieval run launched "
"reported none at /get_server_info; set a number here"
)
elif auto:
# Prelaunch refuses this first; kept as the backstop.
raise ValueError(
f"Model root '{root_key}': concurrency_limit "
f"'{AUTO_CONCURRENCY}' needs an SGLang engine sieval run "
"launched; set a number here"
)
token_budgets[root_key] = budget
connection = CONNECTION_FACTORY_REGISTRY.create(
connection_family,
Expand Down
33 changes: 23 additions & 10 deletions tests/integration/test_auto_concurrency.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
"""Pattern: ``concurrency_limit: auto`` through ``sieval run``, plan to report.

Planning binds each root before its engine exists; the KV pool is read at the
first request. The engines are the stubs of ``test_staged_run`` and answer
``/get_server_info`` once launched.
first request; without ``auto`` none is read. The engines are the stubs of
``test_staged_run`` and answer ``/get_server_info`` once launched.

AI-Generated Code - Claude Opus 5.5 (Anthropic)
"""
Expand Down Expand Up @@ -118,10 +118,11 @@ async def test_a_run_reads_the_pool_once_the_engine_is_up(
reports = await _run(_sglang(_write(tmp_path, subject=AUTO)))

assert reports["judged"]["correct"] == 3
budgets = _budgets()
adaptive = [b for b in budgets if isinstance(b, AdaptiveTokenBudget)]
assert len(adaptive) == 1
assert adaptive[0].total_tokens == 500_000
# Only the subject sets `auto`.
(budget,) = _budgets()
assert isinstance(budget, AdaptiveTokenBudget)
assert budget.total_tokens == 500_000
assert facts.asked == ["sglang"]

@pytest.mark.anyio
async def test_a_judge_started_at_a_barrier_gets_auto_after_it_is_up(
Expand All @@ -142,7 +143,20 @@ async def test_a_judge_started_at_a_barrier_gets_auto_after_it_is_up(
assert [b.total_tokens for b in adaptive] == [500_000]

@pytest.mark.anyio
async def test_a_judge_with_no_limit_started_at_a_barrier_gets_kv_admission(
async def test_no_limit_gets_no_kv_admission(
self,
tmp_path: Path,
engines: Engines,
facts: KvFacts,
) -> None:
reports = await _run(_sglang(_write(tmp_path)))

assert reports["judged"]["correct"] == 3
assert facts.asked == []
assert _budgets() == []

@pytest.mark.anyio
async def test_a_judge_with_no_limit_started_at_a_barrier_gets_no_kv_admission(
self,
tmp_path: Path,
engines: Engines,
Expand All @@ -153,9 +167,8 @@ async def test_a_judge_with_no_limit_started_at_a_barrier_gets_kv_admission(
reports = await _run(config)

assert reports["judged"]["correct"] == 3
# Plain KV admission, as an engine started up front gets (#277).
plain = [b for b in _budgets() if not isinstance(b, AdaptiveTokenBudget)]
assert [b.total_tokens for b in plain] == [500_000, 500_000]
assert facts.asked == []
assert _budgets() == []

@pytest.mark.anyio
async def test_auto_on_an_engine_that_reports_no_pool_is_refused(
Expand Down
Loading
Loading