diff --git a/CHANGELOG.md b/CHANGELOG.md index de898cf9..025e6d6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- **KV admission now needs `concurrency_limit: auto`.** Since 0.15.5 (#277) a root with no `concurrency_limit` on an SGLang engine `sieval run` launched was admitted by KV cache; it is now bounded only by its task and stage limits, as before 0.15.5 (no `KV:` line, no `kv_tokens`). Set `concurrency_limit: auto` to keep KV admission. Throughput only: identity schema and `RESULT_SCHEMA_VERSION` are unchanged, and 0.15.6 result directories resume. + ## [0.15.6] - 2026-11-08 A patch release: `concurrency_limit: auto` (#278) works again. 0.15.5's diff --git a/docs/guide/concurrency.md b/docs/guide/concurrency.md index 5c5d0d2b..b0a8101f 100644 --- a/docs/guide/concurrency.md +++ b/docs/guide/concurrency.md @@ -16,31 +16,12 @@ config = TaskRunnerConfig(concurrency_limits={"infer": 64, "postprocess": 32}) model = ChatModel("gpt-4o", concurrency_limit=32) ``` -## KV Admission - -An SGLang engine `sieval run` launches also admits requests by the KV cache -they may hold, unless its root model sets `concurrency_limit`. - -- **Capacity.** `max_total_num_tokens` from `/get_server_info` once the - engine is ready, times its data-parallel ranks. The launch prints - `KV: x `. vLLM, external endpoints and an engine that - does not report it get none. -- **Size.** `n × min(context, max_tokens + 4096)`, a whole context per - sequence when `max_tokens` is unset. One `n: 8` request holds what eight - `n: 1` ones do; a 1024-token request holds 5120. -- **Order.** First come, first served, after the other limits. A request - waits before it is sent, so no client timeout runs meanwhile. One larger - than the whole pool runs alone. -- Throughput only: no stage identity changes. Request records show the - budget as `kv_tokens` under `in_flight`. - ## `concurrency_limit: auto` A root model on an SGLang engine `sieval run` launches can set -`concurrency_limit: auto`: KV admission as above, and while requests wait, -SiEval reads the engine's `/v1/loads` once a second. A waiting request is held -back while the engine reports queued requests or a KV pool at least 95% full. -An engine that does not answer leaves the KV budget in charge. +`concurrency_limit: auto` to admit requests by the KV cache they may hold. +Only `auto` does this: a number is a fixed cap, and no limit leaves the root +bounded by the task and stage limits alone. ```yaml models: @@ -51,6 +32,21 @@ models: checkpoint: /models/candidate ``` +- **Capacity.** `max_total_num_tokens` from `/get_server_info` once the + engine is ready, times its data-parallel ranks. The launch prints + `KV: x `. +- **Size.** `n × min(context, max_tokens + 4096)`, a whole context per + sequence when `max_tokens` is unset. One `n: 8` request holds what eight + `n: 1` ones do; a 1024-token request holds 5120. +- **Order.** First come, first served, after the other limits. A request + waits before it is sent, so no client timeout runs meanwhile. One larger + than the whole pool runs alone. +- **Engine load.** While requests wait, SiEval polls `/v1/loads` once a + second and holds them back while the engine reports queued requests or a + KV pool at least 95% full. No answer leaves the KV budget in charge. +- Throughput only: no stage identity changes. Request records show the + budget as `kv_tokens` under `in_flight`. + `--dry-run` accepts it before the engine exists; a model started at a barrier reads its pool at its first request. If the engine reports no KV pool, its requests fail naming the root. Anywhere else (external API, vLLM, a derived diff --git a/sieval/cli/run.py b/sieval/cli/run.py index f9f779ab..b549c681 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -92,6 +92,7 @@ read_persisted_config, redact_secrets, resolve_deterministic, + sets_auto_concurrency, unwrap_proxies, ) from sieval.cli.signals import terminate_as_interrupt @@ -1506,7 +1507,12 @@ class _Engines: deployments: dict[str, Deployment] = dataclasses.field(default_factory=dict) async def start( - self, name: str, plan: DeploymentPlan, commands: list[BackendCommand] + self, + name: str, + plan: DeploymentPlan, + commands: list[BackendCommand], + *, + auto: bool, ) -> Deployment: if name in self.launched: return self.deployments[name] @@ -1514,6 +1520,7 @@ async def start( name, plan, commands, + auto=auto, deployer=self.deployer, launched=self.launched, handle_dir=self.handle_dir, @@ -1592,10 +1599,17 @@ async def _run_group( await check_launch_gpu(required=True) for root in sorted(group.own - runnable): log_user("No work for model {} in this run; not starting it", group.names[root]) + models = config.get("models") or {} + + def auto(root: str) -> bool: + return sets_auto_concurrency(models.get(root) or {}) + realized: dict[str, Deployment] = {} for root in sorted(upfront): plan, commands = group.prepared[root] - realized[root] = await engines.start(group.names[root], plan, commands) + realized[root] = await engines.start( + group.names[root], plan, commands, auto=auto(root) + ) for root in sorted(deferred): # Up already, kept from an earlier group. name = group.names[root] @@ -1616,7 +1630,7 @@ async def on_barrier(stage: str) -> None: for root in sorted(deferred): if staged.starts(root) == stage: plan, commands = group.prepared[root] - await engines.start(group.names[root], plan, commands) + await engines.start(group.names[root], plan, commands, auto=auto(root)) watches.start(root, engines.launched[group.names[root]]) serving = dataclasses.replace(serving, on_barrier=on_barrier) @@ -1665,6 +1679,7 @@ async def _launch( plan: DeploymentPlan, commands: list[BackendCommand], *, + auto: bool, deployer: LocalDeployer, launched: dict[str, list[InferHandle]], handle_dir: Path, @@ -1673,7 +1688,7 @@ async def _launch( ready_timeout: float, ) -> Deployment: """Start one engine under *model_name*, wait until it is ready, and record - its handles in *launched*.""" + its handles in *launched*; with *auto*, also read its KV pool.""" log_user("Starting inference for model: {}", model_name) last_status = None @@ -1740,6 +1755,8 @@ def _progress(elapsed: float, status_value: str) -> None: env.gpu_count, ) deployment = deployer.build_deployment(plan, new_handles) + if not auto: + return deployment facts = await kv_facts(plan.backend, deployment.api_base) if facts.kv_capacity_tokens is None: return deployment diff --git a/sieval/cli/session.py b/sieval/cli/session.py index e7daa2c0..66f302a6 100644 --- a/sieval/cli/session.py +++ b/sieval/cli/session.py @@ -1257,34 +1257,39 @@ def _default_client_timeout(deployment: Deployment | None) -> httpx.Timeout: ) -#: A root model's ``concurrency_limit`` that adds the engine's own load to KV -#: admission (:class:`AdaptiveTokenBudget`). +#: The root ``concurrency_limit`` that turns on KV admission +#: (:class:`AdaptiveTokenBudget`); no other value does. AUTO_CONCURRENCY = "auto" -def _kv_token_budget( - deployment: Deployment, *, adaptive: bool = False -) -> TokenBudget | None: - """KV admission for an engine ``sieval run`` launched that reported - its pool; ``None`` otherwise. *adaptive* also reads an SGLang engine's - load, and is ``None`` on any other engine.""" - return _budget_from_facts(deployment, deployment.facts, adaptive=adaptive) +def _configured_concurrency_limit( + config: Mapping[str, object], nested: Mapping[str, object] +) -> object: + """``concurrency_limit``: the top level wins over ``args``.""" + return config.get("concurrency_limit", nested.get("concurrency_limit")) + + +def sets_auto_concurrency(model_config: Mapping[str, object]) -> bool: + """Whether a root model sets ``concurrency_limit: auto``.""" + limit = _configured_concurrency_limit(model_config, _nested_args(model_config)) + return limit == AUTO_CONCURRENCY + + +def _kv_token_budget(deployment: Deployment) -> TokenBudget | None: + """KV admission from the pool a launched SGLang engine reported, else ``None``.""" + return _budget_from_facts(deployment, deployment.facts) def _pending_kv_token_budget( - deployment: Deployment, *, root: str, adaptive: bool + deployment: Deployment, *, root: str ) -> PendingTokenBudget: - """KV admission for an SGLang engine not up yet, read at its first request. - - The launch waits for readiness, so this is the same one read an engine - started up front gets. With no pool reported, ``auto`` fails naming the - root, and no limit runs without KV admission. - """ + """KV admission for an SGLang engine not up yet, read at its first request; + fails naming *root* if the engine reports no pool.""" async def resolve() -> TokenBudget | None: facts = await kv_facts(deployment.engine.engine_id, deployment.api_base) - budget = _budget_from_facts(deployment, facts, adaptive=adaptive) - if adaptive and budget is None: + budget = _budget_from_facts(deployment, facts) + if budget is None: raise RuntimeError( f"Model root '{root}': concurrency_limit '{AUTO_CONCURRENCY}' " "needs the engine's KV pool and context window, and the SGLang " @@ -1297,23 +1302,17 @@ async def resolve() -> TokenBudget | None: def _budget_from_facts( - deployment: Deployment, facts: ServingFacts, *, adaptive: bool + deployment: Deployment, facts: ServingFacts ) -> TokenBudget | None: if ( deployment.plan is None + or deployment.engine.engine_id != "sglang" or facts.kv_capacity_tokens is None or facts.context_tokens is None ): return None - capacity = facts.kv_capacity_tokens * facts.replicas - if not adaptive: - return TokenBudget( - capacity, context=facts.context_tokens, headroom=PROMPT_HEADROOM - ) - if deployment.engine.engine_id != "sglang": - return None return AdaptiveTokenBudget( - capacity, + facts.kv_capacity_tokens * facts.replicas, context=facts.context_tokens, headroom=PROMPT_HEADROOM, probe=SglangLoadProbe(deployment.api_base), @@ -3972,9 +3971,7 @@ def _connection_options( config, nested, subject, os.environ if environ is None else environ ) max_retries = config.get("max_retries", nested.get("max_retries", 3)) - concurrency_limit = config.get( - "concurrency_limit", nested.get("concurrency_limit") - ) + concurrency_limit = _configured_concurrency_limit(config, nested) if isinstance(max_retries, bool) or not isinstance(max_retries, int): raise TypeError("max_retries must be an integer") if concurrency_limit != AUTO_CONCURRENCY and ( @@ -4269,31 +4266,25 @@ def _create_owned_pool( token_budgets = {} self._root_token_budgets = token_budgets if root_key not in token_budgets: - auto = concurrency_limit == AUTO_CONCURRENCY - budget: TokenBudget | PendingTokenBudget | None = ( - _kv_token_budget(deployment, adaptive=auto) - if auto or concurrency_limit is None - else None - ) - if budget is None and (auto or concurrency_limit is None): - if _launches_sglang(deployment) and self._not_up_yet(root_key): - budget = _pending_kv_token_budget( - deployment, root=root_key, adaptive=auto + budget: TokenBudget | PendingTokenBudget | None = None + if concurrency_limit == AUTO_CONCURRENCY: + budget = _kv_token_budget(deployment) + if budget is None and not _launches_sglang(deployment): + # Prelaunch refuses this first; kept as the backstop. + raise ValueError( + f"Model root '{root_key}': concurrency_limit " + f"'{AUTO_CONCURRENCY}' needs an SGLang engine sieval run " + "launched; set a number here" ) - elif auto and _launches_sglang(deployment): + if budget is None and self._not_up_yet(root_key): + budget = _pending_kv_token_budget(deployment, root=root_key) + if budget is None: raise ValueError( f"Model root '{root_key}': concurrency_limit " f"'{AUTO_CONCURRENCY}' needs the KV pool and context " "window, and the SGLang engine sieval run launched " "reported none at /get_server_info; set a number here" ) - elif auto: - # Prelaunch refuses this first; kept as the backstop. - raise ValueError( - f"Model root '{root_key}': concurrency_limit " - f"'{AUTO_CONCURRENCY}' needs an SGLang engine sieval run " - "launched; set a number here" - ) token_budgets[root_key] = budget connection = CONNECTION_FACTORY_REGISTRY.create( connection_family, diff --git a/tests/integration/test_auto_concurrency.py b/tests/integration/test_auto_concurrency.py index 44574e70..bf123718 100644 --- a/tests/integration/test_auto_concurrency.py +++ b/tests/integration/test_auto_concurrency.py @@ -1,8 +1,8 @@ """Pattern: ``concurrency_limit: auto`` through ``sieval run``, plan to report. Planning binds each root before its engine exists; the KV pool is read at the -first request. The engines are the stubs of ``test_staged_run`` and answer -``/get_server_info`` once launched. +first request; without ``auto`` none is read. The engines are the stubs of +``test_staged_run`` and answer ``/get_server_info`` once launched. AI-Generated Code - Claude Opus 5.5 (Anthropic) """ @@ -118,10 +118,11 @@ async def test_a_run_reads_the_pool_once_the_engine_is_up( reports = await _run(_sglang(_write(tmp_path, subject=AUTO))) assert reports["judged"]["correct"] == 3 - budgets = _budgets() - adaptive = [b for b in budgets if isinstance(b, AdaptiveTokenBudget)] - assert len(adaptive) == 1 - assert adaptive[0].total_tokens == 500_000 + # Only the subject sets `auto`. + (budget,) = _budgets() + assert isinstance(budget, AdaptiveTokenBudget) + assert budget.total_tokens == 500_000 + assert facts.asked == ["sglang"] @pytest.mark.anyio async def test_a_judge_started_at_a_barrier_gets_auto_after_it_is_up( @@ -142,7 +143,20 @@ async def test_a_judge_started_at_a_barrier_gets_auto_after_it_is_up( assert [b.total_tokens for b in adaptive] == [500_000] @pytest.mark.anyio - async def test_a_judge_with_no_limit_started_at_a_barrier_gets_kv_admission( + async def test_no_limit_gets_no_kv_admission( + self, + tmp_path: Path, + engines: Engines, + facts: KvFacts, + ) -> None: + reports = await _run(_sglang(_write(tmp_path))) + + assert reports["judged"]["correct"] == 3 + assert facts.asked == [] + assert _budgets() == [] + + @pytest.mark.anyio + async def test_a_judge_with_no_limit_started_at_a_barrier_gets_no_kv_admission( self, tmp_path: Path, engines: Engines, @@ -153,9 +167,8 @@ async def test_a_judge_with_no_limit_started_at_a_barrier_gets_kv_admission( reports = await _run(config) assert reports["judged"]["correct"] == 3 - # Plain KV admission, as an engine started up front gets (#277). - plain = [b for b in _budgets() if not isinstance(b, AdaptiveTokenBudget)] - assert [b.total_tokens for b in plain] == [500_000, 500_000] + assert facts.asked == [] + assert _budgets() == [] @pytest.mark.anyio async def test_auto_on_an_engine_that_reports_no_pool_is_refused( diff --git a/tests/unit/cli/test_session.py b/tests/unit/cli/test_session.py index 856be367..beebc9cd 100644 --- a/tests/unit/cli/test_session.py +++ b/tests/unit/cli/test_session.py @@ -10721,7 +10721,7 @@ def test_no_facts_or_not_launched_gives_none(self) -> None: class TestAutoConcurrency: - """`concurrency_limit: auto` is for a launched SGLang that reported its KV.""" + """Only `concurrency_limit: auto` gets KV admission, on a launched SGLang.""" def test_auto_on_a_launched_sglang_gets_adaptive_admission(self) -> None: from sieval.cli.session import _kv_token_budget @@ -10730,35 +10730,104 @@ def test_auto_on_a_launched_sglang_gets_adaptive_admission(self) -> None: deployment = TestKvTokenBudget._deployment( kv_capacity_tokens=1000, context_tokens=100 ) - assert isinstance( - _kv_token_budget(deployment, adaptive=True), AdaptiveTokenBudget - ) - assert not isinstance(_kv_token_budget(deployment), AdaptiveTokenBudget) + assert isinstance(_kv_token_budget(deployment), AdaptiveTokenBudget) - def test_auto_without_kv_facts_launch_or_sglang_gets_none(self) -> None: + def test_no_kv_facts_launch_or_sglang_gets_none(self) -> None: from dataclasses import replace from sieval.cli.session import _kv_token_budget - from sieval.core.models import Engine - - assert _kv_token_budget(TestKvTokenBudget._deployment(), adaptive=True) is None - assert ( - _kv_token_budget( - TestKvTokenBudget._deployment( - launched=False, kv_capacity_tokens=1, context_tokens=1 - ), - adaptive=True, - ) - is None - ) - from sieval.core.models import DeploymentPlanProjection + from sieval.core.models import DeploymentPlanProjection, Engine vllm = replace( TestKvTokenBudget._deployment(kv_capacity_tokens=1, context_tokens=1), plan=DeploymentPlanProjection("fp", "vllm"), engine=Engine("vllm"), ) - assert _kv_token_budget(vllm, adaptive=True) is None + assert _kv_token_budget(vllm) is None + + @pytest.mark.parametrize( + ("config", "auto"), + [ + ({}, False), + ({"concurrency_limit": 4}, False), + ({"concurrency_limit": "auto"}, True), + ({"args": {"concurrency_limit": "auto"}}, True), + ({"args": {"concurrency_limit": 4}}, False), + # The top level wins, as in `_connection_options`. + ({"concurrency_limit": 4, "args": {"concurrency_limit": "auto"}}, False), + ], + ) + def test_sets_auto_concurrency_reads_both_spellings( + self, config: dict[str, object], auto: bool + ) -> None: + from sieval.cli.session import sets_auto_concurrency + + assert sets_auto_concurrency(config) is auto + + @staticmethod + def _pool_for(config: dict[str, object], **facts: Any): + """The pool a root on a launched SGLang engine gets for *config*.""" + from sieval.core.models import ConnectionIdentity, ResolvedRoute + from sieval.core.models.reconcile import RuntimeBindingPlan + + session = EvalSession.__new__(EvalSession) + session._owned_pools = {} + session._realized_deployments_by_root = {} + identity = ConnectionIdentity( + endpoint="http://127.0.0.1:1/v1", + connection_family="openai_sdk", + credential_scope="model:m:managed-local-credential", + retry_policy="openai-sdk:max-retries=3", + quota_scope="model:m", + ) + runtime_plan = cast( + RuntimeBindingPlan, + types.SimpleNamespace( + dialect_id="openai_chat", + resolved_route=ResolvedRoute( + service_role="default", + endpoint=identity.endpoint, + connection_family="openai_sdk", + fingerprint="route", + ), + connection_identity=identity, + ), + ) + with patch("sieval.core.models.connection_factory.AsyncOpenAI"): + return session._create_owned_pool( + "model:m", + TestKvTokenBudget._deployment(**facts), + runtime_plan, + {"name": "org/m", **config}, + nested_args=True, + ) + + @pytest.mark.parametrize("config", [{}, {"concurrency_limit": 4}]) + def test_no_auto_gets_no_kv_admission(self, config: dict[str, object]) -> None: + # The engine reported its pool; only `auto` admits by it. + pool = self._pool_for(config, kv_capacity_tokens=500_000, context_tokens=32_768) + + assert pool.token_budget is None + limit = config.get("concurrency_limit") + shared = pool.shared_limiter + if limit is None: + assert shared is None + else: + assert isinstance(shared, anyio.CapacityLimiter) + assert shared.total_tokens == limit + + def test_auto_builds_the_kv_budget(self) -> None: + from sieval.core.models.admission import AdaptiveTokenBudget + + pool = self._pool_for( + {"concurrency_limit": "auto"}, + kv_capacity_tokens=500_000, + context_tokens=32_768, + ) + + assert isinstance(pool.token_budget, AdaptiveTokenBudget) + assert pool.token_budget.total_tokens == 500_000 + assert pool.shared_limiter is None @pytest.mark.parametrize("value", ["auto", 4, None]) def test_auto_and_numbers_pass_option_parsing(self, value: object) -> None: