From 28489ce201e00d780213b826878b35496940aaaf Mon Sep 17 00:00:00 2001 From: Ethan Date: Sun, 8 Nov 2026 12:07:40 +0800 Subject: [PATCH 1/2] feat(models)!: KV admission only with concurrency_limit auto KV admission and `concurrency_limit: auto` are now on together or off together. Since #277 an SGLang engine sieval run launched admitted requests by a KV TokenBudget whenever its root set no concurrency_limit; `auto` added the engine's load on top (#278), and #284 deferred either to the first request for a root bound before its engine was up. A root with no concurrency_limit is back on the path it had before #277: no shared root limiter, no token budget, and the launch reads no /get_server_info, so it prints no `KV:` line and its request records carry no `kv_tokens`. A number keeps its fixed cap. `auto` is unchanged: an AdaptiveTokenBudget from the pool the launch read, or a PendingTokenBudget read at the first request when the engine is not up yet, and the same refusals when no pool is reported. No identity or plan field depended on the default: the KV facts are not in the deployment fingerprint, and admission was throughput only. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 + docs/guide/concurrency.md | 42 ++++---- sieval/cli/run.py | 26 ++++- sieval/cli/session.py | 82 ++++++++-------- tests/integration/test_auto_concurrency.py | 33 +++++-- tests/unit/cli/test_session.py | 109 +++++++++++++++++---- 6 files changed, 199 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de898cf9..5537ffb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- **KV admission is `concurrency_limit: auto`'s alone.** Since 0.15.5 (#277), an SGLang engine `sieval run` launches admitted requests by the KV cache they may hold whenever its root model set no `concurrency_limit`. Now only `auto` does: a root with no `concurrency_limit` runs bounded by its task and stage limits, as before 0.15.5, and the launch reads no KV pool, prints no `KV:` line, and its request records carry no `kv_tokens`. Set `concurrency_limit: auto` for KV admission. Throughput only: no on-disk format changes, `meta.json` `identity` stays schema 5, `RESULT_SCHEMA_VERSION` stays 1, and 0.15.6 result directories resume. + ## [0.15.6] - 2026-11-08 A patch release: `concurrency_limit: auto` (#278) works again. 0.15.5's diff --git a/docs/guide/concurrency.md b/docs/guide/concurrency.md index 5c5d0d2b..3ed418de 100644 --- a/docs/guide/concurrency.md +++ b/docs/guide/concurrency.md @@ -16,31 +16,13 @@ config = TaskRunnerConfig(concurrency_limits={"infer": 64, "postprocess": 32}) model = ChatModel("gpt-4o", concurrency_limit=32) ``` -## KV Admission - -An SGLang engine `sieval run` launches also admits requests by the KV cache -they may hold, unless its root model sets `concurrency_limit`. - -- **Capacity.** `max_total_num_tokens` from `/get_server_info` once the - engine is ready, times its data-parallel ranks. The launch prints - `KV: x `. vLLM, external endpoints and an engine that - does not report it get none. -- **Size.** `n × min(context, max_tokens + 4096)`, a whole context per - sequence when `max_tokens` is unset. One `n: 8` request holds what eight - `n: 1` ones do; a 1024-token request holds 5120. -- **Order.** First come, first served, after the other limits. A request - waits before it is sent, so no client timeout runs meanwhile. One larger - than the whole pool runs alone. -- Throughput only: no stage identity changes. Request records show the - budget as `kv_tokens` under `in_flight`. - ## `concurrency_limit: auto` A root model on an SGLang engine `sieval run` launches can set -`concurrency_limit: auto`: KV admission as above, and while requests wait, -SiEval reads the engine's `/v1/loads` once a second. A waiting request is held -back while the engine reports queued requests or a KV pool at least 95% full. -An engine that does not answer leaves the KV budget in charge. +`concurrency_limit: auto`. Its requests are then admitted by the KV cache they +may hold, and held back while the engine reports itself congested. Without +`auto`, a root has no KV admission: no `concurrency_limit` leaves it bounded +only by the task and stage limits, and a number is a fixed cap. ```yaml models: @@ -51,6 +33,22 @@ models: checkpoint: /models/candidate ``` +- **Capacity.** `max_total_num_tokens` from `/get_server_info` once the + engine is ready, times its data-parallel ranks. The launch prints + `KV: x `. +- **Size.** `n × min(context, max_tokens + 4096)`, a whole context per + sequence when `max_tokens` is unset. One `n: 8` request holds what eight + `n: 1` ones do; a 1024-token request holds 5120. +- **Order.** First come, first served, after the other limits. A request + waits before it is sent, so no client timeout runs meanwhile. One larger + than the whole pool runs alone. +- **Engine load.** While requests wait, SiEval reads the engine's `/v1/loads` + once a second. A waiting request that fits is held back while the engine + reports queued requests or a KV pool at least 95% full. An engine that does + not answer leaves the KV budget in charge. +- Throughput only: no stage identity changes. Request records show the + budget as `kv_tokens` under `in_flight`. + `--dry-run` accepts it before the engine exists; a model started at a barrier reads its pool at its first request. If the engine reports no KV pool, its requests fail naming the root. Anywhere else (external API, vLLM, a derived diff --git a/sieval/cli/run.py b/sieval/cli/run.py index f9f779ab..75f1a4df 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -92,6 +92,7 @@ read_persisted_config, redact_secrets, resolve_deterministic, + sets_auto_concurrency, unwrap_proxies, ) from sieval.cli.signals import terminate_as_interrupt @@ -1506,7 +1507,12 @@ class _Engines: deployments: dict[str, Deployment] = dataclasses.field(default_factory=dict) async def start( - self, name: str, plan: DeploymentPlan, commands: list[BackendCommand] + self, + name: str, + plan: DeploymentPlan, + commands: list[BackendCommand], + *, + auto: bool, ) -> Deployment: if name in self.launched: return self.deployments[name] @@ -1514,6 +1520,7 @@ async def start( name, plan, commands, + auto=auto, deployer=self.deployer, launched=self.launched, handle_dir=self.handle_dir, @@ -1592,10 +1599,17 @@ async def _run_group( await check_launch_gpu(required=True) for root in sorted(group.own - runnable): log_user("No work for model {} in this run; not starting it", group.names[root]) + models = config.get("models") or {} + + def auto(root: str) -> bool: + return sets_auto_concurrency(models.get(root) or {}) + realized: dict[str, Deployment] = {} for root in sorted(upfront): plan, commands = group.prepared[root] - realized[root] = await engines.start(group.names[root], plan, commands) + realized[root] = await engines.start( + group.names[root], plan, commands, auto=auto(root) + ) for root in sorted(deferred): # Up already, kept from an earlier group. name = group.names[root] @@ -1616,7 +1630,7 @@ async def on_barrier(stage: str) -> None: for root in sorted(deferred): if staged.starts(root) == stage: plan, commands = group.prepared[root] - await engines.start(group.names[root], plan, commands) + await engines.start(group.names[root], plan, commands, auto=auto(root)) watches.start(root, engines.launched[group.names[root]]) serving = dataclasses.replace(serving, on_barrier=on_barrier) @@ -1665,6 +1679,7 @@ async def _launch( plan: DeploymentPlan, commands: list[BackendCommand], *, + auto: bool, deployer: LocalDeployer, launched: dict[str, list[InferHandle]], handle_dir: Path, @@ -1673,7 +1688,8 @@ async def _launch( ready_timeout: float, ) -> Deployment: """Start one engine under *model_name*, wait until it is ready, and record - its handles in *launched*.""" + its handles in *launched*. With *auto* (its root sets ``concurrency_limit: + auto``), also read the KV pool its admission needs.""" log_user("Starting inference for model: {}", model_name) last_status = None @@ -1740,6 +1756,8 @@ def _progress(elapsed: float, status_value: str) -> None: env.gpu_count, ) deployment = deployer.build_deployment(plan, new_handles) + if not auto: + return deployment facts = await kv_facts(plan.backend, deployment.api_base) if facts.kv_capacity_tokens is None: return deployment diff --git a/sieval/cli/session.py b/sieval/cli/session.py index e7daa2c0..eae81bcb 100644 --- a/sieval/cli/session.py +++ b/sieval/cli/session.py @@ -1257,34 +1257,43 @@ def _default_client_timeout(deployment: Deployment | None) -> httpx.Timeout: ) -#: A root model's ``concurrency_limit`` that adds the engine's own load to KV -#: admission (:class:`AdaptiveTokenBudget`). +#: A root model's ``concurrency_limit`` that turns on KV admission with the +#: engine's own load (:class:`AdaptiveTokenBudget`). Without it a root has no +#: KV admission. AUTO_CONCURRENCY = "auto" -def _kv_token_budget( - deployment: Deployment, *, adaptive: bool = False -) -> TokenBudget | None: - """KV admission for an engine ``sieval run`` launched that reported - its pool; ``None`` otherwise. *adaptive* also reads an SGLang engine's - load, and is ``None`` on any other engine.""" - return _budget_from_facts(deployment, deployment.facts, adaptive=adaptive) +def sets_auto_concurrency(model_config: Mapping[str, object]) -> bool: + """Whether a root model sets ``concurrency_limit: auto``, at its top + level or under ``args``.""" + args = model_config.get("args") + nested: Mapping[str, object] = ( + cast(Mapping[str, object], args) if isinstance(args, Mapping) else {} + ) + limit = model_config.get("concurrency_limit", nested.get("concurrency_limit")) + return limit == AUTO_CONCURRENCY + + +def _kv_token_budget(deployment: Deployment) -> TokenBudget | None: + """``auto``'s KV admission for an SGLang engine ``sieval run`` launched + that reported its pool; ``None`` otherwise.""" + return _budget_from_facts(deployment, deployment.facts) def _pending_kv_token_budget( - deployment: Deployment, *, root: str, adaptive: bool + deployment: Deployment, *, root: str ) -> PendingTokenBudget: - """KV admission for an SGLang engine not up yet, read at its first request. + """``auto``'s KV admission for an SGLang engine not up yet, read at its + first request. The launch waits for readiness, so this is the same one read an engine - started up front gets. With no pool reported, ``auto`` fails naming the - root, and no limit runs without KV admission. + started up front gets. With no pool reported, it fails naming the root. """ async def resolve() -> TokenBudget | None: facts = await kv_facts(deployment.engine.engine_id, deployment.api_base) - budget = _budget_from_facts(deployment, facts, adaptive=adaptive) - if adaptive and budget is None: + budget = _budget_from_facts(deployment, facts) + if budget is None: raise RuntimeError( f"Model root '{root}': concurrency_limit '{AUTO_CONCURRENCY}' " "needs the engine's KV pool and context window, and the SGLang " @@ -1297,23 +1306,17 @@ async def resolve() -> TokenBudget | None: def _budget_from_facts( - deployment: Deployment, facts: ServingFacts, *, adaptive: bool + deployment: Deployment, facts: ServingFacts ) -> TokenBudget | None: if ( deployment.plan is None + or deployment.engine.engine_id != "sglang" or facts.kv_capacity_tokens is None or facts.context_tokens is None ): return None - capacity = facts.kv_capacity_tokens * facts.replicas - if not adaptive: - return TokenBudget( - capacity, context=facts.context_tokens, headroom=PROMPT_HEADROOM - ) - if deployment.engine.engine_id != "sglang": - return None return AdaptiveTokenBudget( - capacity, + facts.kv_capacity_tokens * facts.replicas, context=facts.context_tokens, headroom=PROMPT_HEADROOM, probe=SglangLoadProbe(deployment.api_base), @@ -4269,31 +4272,26 @@ def _create_owned_pool( token_budgets = {} self._root_token_budgets = token_budgets if root_key not in token_budgets: - auto = concurrency_limit == AUTO_CONCURRENCY - budget: TokenBudget | PendingTokenBudget | None = ( - _kv_token_budget(deployment, adaptive=auto) - if auto or concurrency_limit is None - else None - ) - if budget is None and (auto or concurrency_limit is None): - if _launches_sglang(deployment) and self._not_up_yet(root_key): - budget = _pending_kv_token_budget( - deployment, root=root_key, adaptive=auto + # KV admission is `auto`'s alone; no limit or a number has none. + budget: TokenBudget | PendingTokenBudget | None = None + if concurrency_limit == AUTO_CONCURRENCY: + budget = _kv_token_budget(deployment) + if budget is None and not _launches_sglang(deployment): + # Prelaunch refuses this first; kept as the backstop. + raise ValueError( + f"Model root '{root_key}': concurrency_limit " + f"'{AUTO_CONCURRENCY}' needs an SGLang engine sieval run " + "launched; set a number here" ) - elif auto and _launches_sglang(deployment): + if budget is None and self._not_up_yet(root_key): + budget = _pending_kv_token_budget(deployment, root=root_key) + if budget is None: raise ValueError( f"Model root '{root_key}': concurrency_limit " f"'{AUTO_CONCURRENCY}' needs the KV pool and context " "window, and the SGLang engine sieval run launched " "reported none at /get_server_info; set a number here" ) - elif auto: - # Prelaunch refuses this first; kept as the backstop. - raise ValueError( - f"Model root '{root_key}': concurrency_limit " - f"'{AUTO_CONCURRENCY}' needs an SGLang engine sieval run " - "launched; set a number here" - ) token_budgets[root_key] = budget connection = CONNECTION_FACTORY_REGISTRY.create( connection_family, diff --git a/tests/integration/test_auto_concurrency.py b/tests/integration/test_auto_concurrency.py index 44574e70..00d8584a 100644 --- a/tests/integration/test_auto_concurrency.py +++ b/tests/integration/test_auto_concurrency.py @@ -1,7 +1,8 @@ """Pattern: ``concurrency_limit: auto`` through ``sieval run``, plan to report. Planning binds each root before its engine exists; the KV pool is read at the -first request. The engines are the stubs of ``test_staged_run`` and answer +first request. Without ``auto`` no pool is read and nothing is admitted by KV. +The engines are the stubs of ``test_staged_run`` and answer ``/get_server_info`` once launched. AI-Generated Code - Claude Opus 5.5 (Anthropic) @@ -118,10 +119,11 @@ async def test_a_run_reads_the_pool_once_the_engine_is_up( reports = await _run(_sglang(_write(tmp_path, subject=AUTO))) assert reports["judged"]["correct"] == 3 - budgets = _budgets() - adaptive = [b for b in budgets if isinstance(b, AdaptiveTokenBudget)] - assert len(adaptive) == 1 - assert adaptive[0].total_tokens == 500_000 + # Only the subject sets `auto`; the judge, with no limit, has no budget. + (budget,) = _budgets() + assert isinstance(budget, AdaptiveTokenBudget) + assert budget.total_tokens == 500_000 + assert facts.asked == ["sglang"] @pytest.mark.anyio async def test_a_judge_started_at_a_barrier_gets_auto_after_it_is_up( @@ -142,7 +144,21 @@ async def test_a_judge_started_at_a_barrier_gets_auto_after_it_is_up( assert [b.total_tokens for b in adaptive] == [500_000] @pytest.mark.anyio - async def test_a_judge_with_no_limit_started_at_a_barrier_gets_kv_admission( + async def test_no_limit_gets_no_kv_admission( + self, + tmp_path: Path, + engines: Engines, + facts: KvFacts, + ) -> None: + reports = await _run(_sglang(_write(tmp_path))) + + assert reports["judged"]["correct"] == 3 + # KV admission is `auto`'s: no pool is read, no budget is reserved. + assert facts.asked == [] + assert _budgets() == [] + + @pytest.mark.anyio + async def test_a_judge_with_no_limit_started_at_a_barrier_gets_no_kv_admission( self, tmp_path: Path, engines: Engines, @@ -153,9 +169,8 @@ async def test_a_judge_with_no_limit_started_at_a_barrier_gets_kv_admission( reports = await _run(config) assert reports["judged"]["correct"] == 3 - # Plain KV admission, as an engine started up front gets (#277). - plain = [b for b in _budgets() if not isinstance(b, AdaptiveTokenBudget)] - assert [b.total_tokens for b in plain] == [500_000, 500_000] + assert facts.asked == [] + assert _budgets() == [] @pytest.mark.anyio async def test_auto_on_an_engine_that_reports_no_pool_is_refused( diff --git a/tests/unit/cli/test_session.py b/tests/unit/cli/test_session.py index 856be367..d887d9dd 100644 --- a/tests/unit/cli/test_session.py +++ b/tests/unit/cli/test_session.py @@ -10721,7 +10721,8 @@ def test_no_facts_or_not_launched_gives_none(self) -> None: class TestAutoConcurrency: - """`concurrency_limit: auto` is for a launched SGLang that reported its KV.""" + """KV admission is `concurrency_limit: auto`'s, on a launched SGLang that + reported its KV.""" def test_auto_on_a_launched_sglang_gets_adaptive_admission(self) -> None: from sieval.cli.session import _kv_token_budget @@ -10730,35 +10731,103 @@ def test_auto_on_a_launched_sglang_gets_adaptive_admission(self) -> None: deployment = TestKvTokenBudget._deployment( kv_capacity_tokens=1000, context_tokens=100 ) - assert isinstance( - _kv_token_budget(deployment, adaptive=True), AdaptiveTokenBudget - ) - assert not isinstance(_kv_token_budget(deployment), AdaptiveTokenBudget) + assert isinstance(_kv_token_budget(deployment), AdaptiveTokenBudget) - def test_auto_without_kv_facts_launch_or_sglang_gets_none(self) -> None: + def test_no_kv_facts_launch_or_sglang_gets_none(self) -> None: from dataclasses import replace from sieval.cli.session import _kv_token_budget - from sieval.core.models import Engine - - assert _kv_token_budget(TestKvTokenBudget._deployment(), adaptive=True) is None - assert ( - _kv_token_budget( - TestKvTokenBudget._deployment( - launched=False, kv_capacity_tokens=1, context_tokens=1 - ), - adaptive=True, - ) - is None - ) - from sieval.core.models import DeploymentPlanProjection + from sieval.core.models import DeploymentPlanProjection, Engine vllm = replace( TestKvTokenBudget._deployment(kv_capacity_tokens=1, context_tokens=1), plan=DeploymentPlanProjection("fp", "vllm"), engine=Engine("vllm"), ) - assert _kv_token_budget(vllm, adaptive=True) is None + assert _kv_token_budget(vllm) is None + + @pytest.mark.parametrize( + ("config", "auto"), + [ + ({}, False), + ({"concurrency_limit": 4}, False), + ({"concurrency_limit": "auto"}, True), + ({"args": {"concurrency_limit": "auto"}}, True), + ({"args": {"concurrency_limit": 4}}, False), + ], + ) + def test_sets_auto_concurrency_reads_both_spellings( + self, config: dict[str, object], auto: bool + ) -> None: + from sieval.cli.session import sets_auto_concurrency + + assert sets_auto_concurrency(config) is auto + + @staticmethod + def _pool_for(config: dict[str, object], **facts: Any): + """The pool a root on a launched SGLang engine gets for *config*.""" + from sieval.core.models import ConnectionIdentity, ResolvedRoute + from sieval.core.models.reconcile import RuntimeBindingPlan + + session = EvalSession.__new__(EvalSession) + session._owned_pools = {} + session._realized_deployments_by_root = {} + identity = ConnectionIdentity( + endpoint="http://127.0.0.1:1/v1", + connection_family="openai_sdk", + credential_scope="model:m:managed-local-credential", + retry_policy="openai-sdk:max-retries=3", + quota_scope="model:m", + ) + runtime_plan = cast( + RuntimeBindingPlan, + types.SimpleNamespace( + dialect_id="openai_chat", + resolved_route=ResolvedRoute( + service_role="default", + endpoint=identity.endpoint, + connection_family="openai_sdk", + fingerprint="route", + ), + connection_identity=identity, + ), + ) + with patch("sieval.core.models.connection_factory.AsyncOpenAI"): + return session._create_owned_pool( + "model:m", + TestKvTokenBudget._deployment(**facts), + runtime_plan, + {"name": "org/m", **config}, + nested_args=True, + ) + + @pytest.mark.parametrize("config", [{}, {"concurrency_limit": 4}]) + def test_no_auto_gets_no_kv_admission(self, config: dict[str, object]) -> None: + # As before KV admission (#277): the engine reported its pool, and + # still only `auto` admits by it. + pool = self._pool_for(config, kv_capacity_tokens=500_000, context_tokens=32_768) + + assert pool.token_budget is None + limit = config.get("concurrency_limit") + shared = pool.shared_limiter + if limit is None: + assert shared is None + else: + assert isinstance(shared, anyio.CapacityLimiter) + assert shared.total_tokens == limit + + def test_auto_builds_the_kv_budget(self) -> None: + from sieval.core.models.admission import AdaptiveTokenBudget + + pool = self._pool_for( + {"concurrency_limit": "auto"}, + kv_capacity_tokens=500_000, + context_tokens=32_768, + ) + + assert isinstance(pool.token_budget, AdaptiveTokenBudget) + assert pool.token_budget.total_tokens == 500_000 + assert pool.shared_limiter is None @pytest.mark.parametrize("value", ["auto", 4, None]) def test_auto_and_numbers_pass_option_parsing(self, value: object) -> None: From 62eb461fcd1e109f4c9f5c889f611132fc701968 Mon Sep 17 00:00:00 2001 From: Ethan Date: Sun, 8 Nov 2026 12:26:59 +0800 Subject: [PATCH 2/2] refactor(session): share the concurrency_limit lookup `sets_auto_concurrency` copied the "top level wins over args" rule from `_connection_options`; if the two drifted, the launch would skip the KV pool read while the session still saw `auto`. Both now call one helper, and a test pins the precedence. Docs, CHANGELOG and comments trimmed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- docs/guide/concurrency.md | 14 ++++---- sieval/cli/run.py | 3 +- sieval/cli/session.py | 37 +++++++++------------- tests/integration/test_auto_concurrency.py | 8 ++--- tests/unit/cli/test_session.py | 8 ++--- 6 files changed, 30 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5537ffb4..025e6d6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed -- **KV admission is `concurrency_limit: auto`'s alone.** Since 0.15.5 (#277), an SGLang engine `sieval run` launches admitted requests by the KV cache they may hold whenever its root model set no `concurrency_limit`. Now only `auto` does: a root with no `concurrency_limit` runs bounded by its task and stage limits, as before 0.15.5, and the launch reads no KV pool, prints no `KV:` line, and its request records carry no `kv_tokens`. Set `concurrency_limit: auto` for KV admission. Throughput only: no on-disk format changes, `meta.json` `identity` stays schema 5, `RESULT_SCHEMA_VERSION` stays 1, and 0.15.6 result directories resume. +- **KV admission now needs `concurrency_limit: auto`.** Since 0.15.5 (#277) a root with no `concurrency_limit` on an SGLang engine `sieval run` launched was admitted by KV cache; it is now bounded only by its task and stage limits, as before 0.15.5 (no `KV:` line, no `kv_tokens`). Set `concurrency_limit: auto` to keep KV admission. Throughput only: identity schema and `RESULT_SCHEMA_VERSION` are unchanged, and 0.15.6 result directories resume. ## [0.15.6] - 2026-11-08 diff --git a/docs/guide/concurrency.md b/docs/guide/concurrency.md index 3ed418de..b0a8101f 100644 --- a/docs/guide/concurrency.md +++ b/docs/guide/concurrency.md @@ -19,10 +19,9 @@ model = ChatModel("gpt-4o", concurrency_limit=32) ## `concurrency_limit: auto` A root model on an SGLang engine `sieval run` launches can set -`concurrency_limit: auto`. Its requests are then admitted by the KV cache they -may hold, and held back while the engine reports itself congested. Without -`auto`, a root has no KV admission: no `concurrency_limit` leaves it bounded -only by the task and stage limits, and a number is a fixed cap. +`concurrency_limit: auto` to admit requests by the KV cache they may hold. +Only `auto` does this: a number is a fixed cap, and no limit leaves the root +bounded by the task and stage limits alone. ```yaml models: @@ -42,10 +41,9 @@ models: - **Order.** First come, first served, after the other limits. A request waits before it is sent, so no client timeout runs meanwhile. One larger than the whole pool runs alone. -- **Engine load.** While requests wait, SiEval reads the engine's `/v1/loads` - once a second. A waiting request that fits is held back while the engine - reports queued requests or a KV pool at least 95% full. An engine that does - not answer leaves the KV budget in charge. +- **Engine load.** While requests wait, SiEval polls `/v1/loads` once a + second and holds them back while the engine reports queued requests or a + KV pool at least 95% full. No answer leaves the KV budget in charge. - Throughput only: no stage identity changes. Request records show the budget as `kv_tokens` under `in_flight`. diff --git a/sieval/cli/run.py b/sieval/cli/run.py index 75f1a4df..b549c681 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -1688,8 +1688,7 @@ async def _launch( ready_timeout: float, ) -> Deployment: """Start one engine under *model_name*, wait until it is ready, and record - its handles in *launched*. With *auto* (its root sets ``concurrency_limit: - auto``), also read the KV pool its admission needs.""" + its handles in *launched*; with *auto*, also read its KV pool.""" log_user("Starting inference for model: {}", model_name) last_status = None diff --git a/sieval/cli/session.py b/sieval/cli/session.py index eae81bcb..66f302a6 100644 --- a/sieval/cli/session.py +++ b/sieval/cli/session.py @@ -1257,38 +1257,34 @@ def _default_client_timeout(deployment: Deployment | None) -> httpx.Timeout: ) -#: A root model's ``concurrency_limit`` that turns on KV admission with the -#: engine's own load (:class:`AdaptiveTokenBudget`). Without it a root has no -#: KV admission. +#: The root ``concurrency_limit`` that turns on KV admission +#: (:class:`AdaptiveTokenBudget`); no other value does. AUTO_CONCURRENCY = "auto" +def _configured_concurrency_limit( + config: Mapping[str, object], nested: Mapping[str, object] +) -> object: + """``concurrency_limit``: the top level wins over ``args``.""" + return config.get("concurrency_limit", nested.get("concurrency_limit")) + + def sets_auto_concurrency(model_config: Mapping[str, object]) -> bool: - """Whether a root model sets ``concurrency_limit: auto``, at its top - level or under ``args``.""" - args = model_config.get("args") - nested: Mapping[str, object] = ( - cast(Mapping[str, object], args) if isinstance(args, Mapping) else {} - ) - limit = model_config.get("concurrency_limit", nested.get("concurrency_limit")) + """Whether a root model sets ``concurrency_limit: auto``.""" + limit = _configured_concurrency_limit(model_config, _nested_args(model_config)) return limit == AUTO_CONCURRENCY def _kv_token_budget(deployment: Deployment) -> TokenBudget | None: - """``auto``'s KV admission for an SGLang engine ``sieval run`` launched - that reported its pool; ``None`` otherwise.""" + """KV admission from the pool a launched SGLang engine reported, else ``None``.""" return _budget_from_facts(deployment, deployment.facts) def _pending_kv_token_budget( deployment: Deployment, *, root: str ) -> PendingTokenBudget: - """``auto``'s KV admission for an SGLang engine not up yet, read at its - first request. - - The launch waits for readiness, so this is the same one read an engine - started up front gets. With no pool reported, it fails naming the root. - """ + """KV admission for an SGLang engine not up yet, read at its first request; + fails naming *root* if the engine reports no pool.""" async def resolve() -> TokenBudget | None: facts = await kv_facts(deployment.engine.engine_id, deployment.api_base) @@ -3975,9 +3971,7 @@ def _connection_options( config, nested, subject, os.environ if environ is None else environ ) max_retries = config.get("max_retries", nested.get("max_retries", 3)) - concurrency_limit = config.get( - "concurrency_limit", nested.get("concurrency_limit") - ) + concurrency_limit = _configured_concurrency_limit(config, nested) if isinstance(max_retries, bool) or not isinstance(max_retries, int): raise TypeError("max_retries must be an integer") if concurrency_limit != AUTO_CONCURRENCY and ( @@ -4272,7 +4266,6 @@ def _create_owned_pool( token_budgets = {} self._root_token_budgets = token_budgets if root_key not in token_budgets: - # KV admission is `auto`'s alone; no limit or a number has none. budget: TokenBudget | PendingTokenBudget | None = None if concurrency_limit == AUTO_CONCURRENCY: budget = _kv_token_budget(deployment) diff --git a/tests/integration/test_auto_concurrency.py b/tests/integration/test_auto_concurrency.py index 00d8584a..bf123718 100644 --- a/tests/integration/test_auto_concurrency.py +++ b/tests/integration/test_auto_concurrency.py @@ -1,9 +1,8 @@ """Pattern: ``concurrency_limit: auto`` through ``sieval run``, plan to report. Planning binds each root before its engine exists; the KV pool is read at the -first request. Without ``auto`` no pool is read and nothing is admitted by KV. -The engines are the stubs of ``test_staged_run`` and answer -``/get_server_info`` once launched. +first request; without ``auto`` none is read. The engines are the stubs of +``test_staged_run`` and answer ``/get_server_info`` once launched. AI-Generated Code - Claude Opus 5.5 (Anthropic) """ @@ -119,7 +118,7 @@ async def test_a_run_reads_the_pool_once_the_engine_is_up( reports = await _run(_sglang(_write(tmp_path, subject=AUTO))) assert reports["judged"]["correct"] == 3 - # Only the subject sets `auto`; the judge, with no limit, has no budget. + # Only the subject sets `auto`. (budget,) = _budgets() assert isinstance(budget, AdaptiveTokenBudget) assert budget.total_tokens == 500_000 @@ -153,7 +152,6 @@ async def test_no_limit_gets_no_kv_admission( reports = await _run(_sglang(_write(tmp_path))) assert reports["judged"]["correct"] == 3 - # KV admission is `auto`'s: no pool is read, no budget is reserved. assert facts.asked == [] assert _budgets() == [] diff --git a/tests/unit/cli/test_session.py b/tests/unit/cli/test_session.py index d887d9dd..beebc9cd 100644 --- a/tests/unit/cli/test_session.py +++ b/tests/unit/cli/test_session.py @@ -10721,8 +10721,7 @@ def test_no_facts_or_not_launched_gives_none(self) -> None: class TestAutoConcurrency: - """KV admission is `concurrency_limit: auto`'s, on a launched SGLang that - reported its KV.""" + """Only `concurrency_limit: auto` gets KV admission, on a launched SGLang.""" def test_auto_on_a_launched_sglang_gets_adaptive_admission(self) -> None: from sieval.cli.session import _kv_token_budget @@ -10754,6 +10753,8 @@ def test_no_kv_facts_launch_or_sglang_gets_none(self) -> None: ({"concurrency_limit": "auto"}, True), ({"args": {"concurrency_limit": "auto"}}, True), ({"args": {"concurrency_limit": 4}}, False), + # The top level wins, as in `_connection_options`. + ({"concurrency_limit": 4, "args": {"concurrency_limit": "auto"}}, False), ], ) def test_sets_auto_concurrency_reads_both_spellings( @@ -10803,8 +10804,7 @@ def _pool_for(config: dict[str, object], **facts: Any): @pytest.mark.parametrize("config", [{}, {"concurrency_limit": 4}]) def test_no_auto_gets_no_kv_admission(self, config: dict[str, object]) -> None: - # As before KV admission (#277): the engine reported its pool, and - # still only `auto` admits by it. + # The engine reported its pool; only `auto` admits by it. pool = self._pool_for(config, kv_capacity_tokens=500_000, context_tokens=32_768) assert pool.token_budget is None