From a3f5a9771535e29e0b938f540bf2079da8e2bfd8 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Nov 2026 17:07:56 +0800 Subject: [PATCH 1/2] fix(serving)!: one predicate chooses and checks a context window A window serves a task when fits(need, window): min_context and a declared need count the prompt (need <= window); a max_tokens need leaves at least one token for it (max_tokens < window). The same rule picks the window sieval run launches and checks an endpoint's context_limit, so a launched engine at native 40960 and an endpoint declaring 40960 now give one verdict on max_tokens 38000: run, with a warning that the prompt has 2960 tokens. Windows chosen are declared ones: the recipe window when it fits every task, else native, else context.max for the scaled group, keyed @ for every task subset. PROMPT_HEADROOM is the warning threshold only. BREAKING CHANGE: a result directory recorded at a need-sized window (a scaled group, or a default group raised to a min_context below native) differs in identity.inference and refuses resume; it needs inference again. Closes #289 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + docs/guide/configuration.md | 7 +- docs/guide/infer.md | 71 +++++--- sieval/cli/run.py | 10 +- sieval/cli/serving.py | 223 ++++++++++++++--------- sieval/cli/session.py | 5 +- sieval/cli/validation.py | 14 +- tests/unit/cli/test_run.py | 97 +++++++++- tests/unit/cli/test_serving.py | 293 ++++++++++++++++++++++-------- tests/unit/cli/test_validation.py | 45 +++++ 10 files changed, 570 insertions(+), 202 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40aadeec..62e22def 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- **One rule decides whether a context window serves a task, whoever picked the window (#289).** A `max_tokens` task fits a window that leaves at least one token for the prompt; `min_context` and a task's declared need still count the prompt and must fit whole. Before, a launched engine at its native window ran `max_tokens: 38000` at 40960, while an endpoint declaring `context_limit: 40960` refused it as needing 42096 (`max_tokens` plus 4096 for the prompt). Both now run it, and both warn, on `--dry-run` and at run time, when the prompt is left under 4096 tokens: `task 't' leaves 2960 tokens for the prompt at window 40960; longer prompts overflow per sample`. A `max_tokens` equal to the window is refused as `max_tokens 32768 leaves no room for the prompt at window 32768`. A longer prompt still overflows per sample (`context_overflow`). +- **A served window is a declared one, never sized to a need.** A scaled group serves `context.max` and is keyed `@`, the same for every subset of tasks; it used to serve, and be keyed by, the largest need among this run's tasks. A `min_context` or declared need between the recipe window and `native` now raises the default group to `native`, not to the need. A recipe window that serves every task still stays, and a window pinned by `infer.overrides` or `context_limit` is still never raised. A scaled engine keeps the recipe's tp; `--dry-run` checks its memory at `context.max` and warns when little KV room is left. Raise tp in `infer.overrides` if one `context.max` sequence does not fit. +- **Resume:** the served window is in `identity.inference`, so a directory recorded at a need-sized window (a scaled group, or a default group raised to a `min_context` below `native`) is refused on `inference` and needs inference again: start fresh, or pin the recorded window with `infer.overrides`. `IDENTITY_SCHEMA` stays 5 and `RESULT_SCHEMA_VERSION` stays 1: no served token changes at a given window, and the window change is already caught by the inference identity. +- `sieval.cli.serving`: `check_context_fits` returns the prompt-room warnings; `task_context_needs` carries a `max_tokens` need as the raw `max_tokens`, no longer plus `PROMPT_HEADROOM`; `fits(need, window)` is the one rule. `PROMPT_HEADROOM` is the warning threshold only. + ## [0.15.7] - 2026-11-08 A patch release: KV admission now needs `concurrency_limit: auto` (#286). diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 3946f9da..6aaefc8e 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -601,9 +601,12 @@ An unknown mode, or `--mode` without `--dry-run`, exits 2. `infer_plans.yaml` would record it, each process the launch starts (`argv`, and `env` and `cwd` when set), and `needs_inference`, whether this run starts its engine. `gpu` is what the plans were resolved for. -- `serving` has one entry per serving group (`default`, or `@`): +- `serving` has one entry per serving group (`default`, or `@`): its `tasks`, their served windows, the `engines` it launches and whether any - starts. The `serving` check fails on a window shortfall. See + starts. The `serving` check fails on a window shortfall, and warns, as + `task '' leaves tokens for the prompt at window ; ...`, for a + `max_tokens` task with under 4096 tokens of prompt room on a launched + engine; an endpoint's goes to `capability_reconcile`. See [Serving plans](infer.md#serving-plans-context-window-and-dialect). An engine serving input scoring has `headroom` ([Input-scoring headroom](infer.md#input-scoring-headroom)). diff --git a/docs/guide/infer.md b/docs/guide/infer.md index aebdc3a9..2ca0494d 100644 --- a/docs/guide/infer.md +++ b/docs/guide/infer.md @@ -225,11 +225,21 @@ SiEval (`api_base`, `sieval infer start`) is not reported. window from what its tasks need, before any engine starts (`--dry-run` included). -A task needs its `min_context` if set, else the larger of what its class +A task needs its `min_context` if set, else the stricter of what its class declares (`TaskRequirements.min_context`; RULER: `max_seq_length` plus any -unreserved thinking budget) and its effective `max_tokens` plus 4096 for the -prompt. Both `min_context` forms already count prompt and output and are used -as they are. A task whose prompts are longer (or known to be shorter) sets +unreserved thinking budget) and its effective `max_tokens`. One rule decides +whether a window serves a task, both when SiEval picks the window and when it +checks one it did not pick: + +- `min_context` (either form) counts prompt and output: the window must be at + least that. +- `max_tokens` counts the output only: the window must exceed it by at least + one token. Whatever is left is the prompt's room; a longer prompt is that + sample's [context overflow](metrics.md#always-present). Room under 4096 + tokens is warned about, naming the task, the room and the window; the run + goes ahead. + +A task whose prompts are longer than the room `max_tokens` leaves sets `min_context`. A root model states what it can serve: ```yaml @@ -248,33 +258,42 @@ tasks: min_context: 38000 # optional: override the derived need ``` -- Within the recipe window: nothing changes. -- Within `native`: the plan's `context_length` / `max_model_len` is raised to - the largest such need, for every task on that model. A `max_tokens` below - `native` raises it to `native` itself, even when the 4096 would not fit: - such a task is never scaled, and its prompt gets `native - max_tokens` - tokens. A longer prompt is that sample's - [context overflow](metrics.md#always-present), as before. -- Within `max`: a plan of its own with `scaling`, in serving group - `@`. Scaling changes short-context outputs, so no other task - shares that engine. SGLang launches it with - `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1`, so `max` is the only bound: - SGLang no longer checks the window against the scaling. `infer.env` - overrides it. +A served window is always a declared one, never sized to a task's need: + +- The recipe window serves every task: nothing changes. +- `native` serves a task the recipe window does not: the plan's + `context_length` / `max_model_len` is raised to `native`, for every task on + that model. +- Only `max` serves it: a plan of its own at `max`, with `scaling`, in serving + group `@`. Every subset of tasks gets the same window, so a + resume across task subsets keeps one inference identity. Scaling changes + short-context outputs, so no other task shares that engine. SGLang launches + it with `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1`, so `max` is the only + bound: SGLang no longer checks the window against the scaling. `infer.env` + overrides it. The engine keeps the recipe's tp; `--dry-run` reports its + memory at `max` under `serving..engines..memory`, with the + low-KV warning when one `max` sequence leaves little room. Raise tp in + `infer.overrides` if it does not fit. - Within `max` with `extrapolate: true` instead of `scaling`: the same group and variable, with no scaling parameters. SGLang only; exclusive with `scaling`. -- Otherwise, or beyond a window pinned by `infer.overrides` / `context_limit`: - refused before launch, naming the task, its need and the model's limit. An - unknown native window refuses any raise (set `context.native`). - -An `api_base` model is checked against its `context_limit`; without one, an -overflow shows per sample. +- Otherwise, or beyond a window pinned by `infer.overrides` / `context_limit` + (never raised): refused before launch, naming the task, what was checked + (`needs 40961 tokens (its min_context)`, or `max_tokens 40960 leaves no room + for the prompt at window 40960`) and the model's limit. An unknown native + window refuses any raise (set `context.native`). + +An `api_base` model is checked against its `context_limit` by the same rule; +without one, an overflow shows per sample. With `max_tokens: 38000` on a model +whose window is 40960, a launched engine and an endpoint declaring +`context_limit: 40960` both run the task and both warn that it leaves 2960 +tokens for the prompt; an endpoint without `context_limit` runs it unchecked. **Resume.** The served window is in each task's inference identity, and a -resume never reruns inference. A run recorded at a window this release now -raises is refused: start fresh, or pin the recorded window with -`infer.overrides`. +resume never reruns inference. A run recorded at a window this release serves +differently (a scaled group recorded at its largest need by an earlier +release, now at `max`) is refused: start fresh, or pin the recorded window +with `infer.overrides`. Groups run one after another, each group's engines stopped before the next start; an engine starts only for a group with tasks on it. `--serving-group diff --git a/sieval/cli/run.py b/sieval/cli/run.py index b549c681..863b4f9f 100644 --- a/sieval/cli/run.py +++ b/sieval/cli/run.py @@ -83,6 +83,7 @@ ServingGroup, SessionServing, plan_serving_groups, + prompt_room_warnings, served_contexts, served_window, ) @@ -1162,7 +1163,9 @@ def _serving_json( Each engine's ``memory`` is how each role fits its GPUs, ``None`` when unknown; a fit with little KV room appends the warning - :func:`~sieval.infer.topology.memory.size_tp` logs to *warnings*. + :func:`~sieval.infer.topology.memory.size_tp` logs to *warnings*, and a + ``max_tokens`` task left little room for its prompt the one + :func:`~sieval.cli.serving.check_context_fits` logs. """ models = config.get("models") or {} engines: dict[str, dict[str, object]] = {} @@ -1191,6 +1194,11 @@ def _serving_json( where = engine_name if len(fits or ()) == 1 else f"{engine_name}.{role}" warnings.append(f"engine {where}: {warning}") engines[engine_name] = engine + warnings.extend( + prompt_room_warnings( + config, group.plan_dicts, frozenset(group.group.tasks), launched=True + ) + ) return { "tasks": list(group.group.tasks), "windows": { diff --git a/sieval/cli/serving.py b/sieval/cli/serving.py index 1c8c0520..821c5044 100644 --- a/sieval/cli/serving.py +++ b/sieval/cli/serving.py @@ -41,10 +41,10 @@ _MAX_TOKENS_KEYS = ("max_tokens", "max_completion_tokens") -#: Prompt room added to a ``max_tokens``-derived need: engines refuse -#: prompt + ``max_tokens`` > window. A ``max_tokens`` below the native window -#: is served at that window instead (:func:`_native_serves`). Longer prompts -#: set ``min_context``. +#: The prompt room below which a ``max_tokens`` task is warned about: engines +#: refuse prompt + ``max_tokens`` > window, so a longer prompt overflows per +#: sample (``context_overflow``). It sizes no window and refuses no run +#: (:func:`fits`); a task whose prompts are longer sets ``min_context``. PROMPT_HEADROOM = 4096 @@ -84,13 +84,31 @@ class ScoringHeadroomError(ValueError): @dataclass(frozen=True, slots=True) class ContextNeed: - """The context window one task needs from its candidate root model.""" + """The context window one task needs from its candidate root model. + + *tokens* counts the prompt for ``min_context`` and ``requires``; for + ``max_tokens`` it is the output alone (:func:`fits`). + """ root: str tokens: int source: str # "min_context", "requires" or "max_tokens" +def fits(need: ContextNeed, window: int) -> bool: + """Whether *window* serves *need*: a need that counts the prompt + (``min_context``, ``requires``) at most *window*; a ``max_tokens`` below it, + leaving at least one token for the prompt. A longer prompt overflows per + sample (``context_overflow``). + + The one rule for choosing a window (:func:`plan_serving_groups`) and for + checking one (:func:`check_context_fits`). + """ + if need.source == "max_tokens": + return need.tokens < window + return need.tokens <= window + + @dataclass(frozen=True, slots=True) class ModelContext: """What a root model's ``context:`` block states. *extrapolate* serves a @@ -274,10 +292,9 @@ def parse_model_context(name: str, config: Mapping[str, Any]) -> ModelContext: def task_context_needs(config: Mapping[str, Any]) -> dict[str, ContextNeed]: """The context window each task needs, by task name. - A need from ``max_tokens`` adds :data:`PROMPT_HEADROOM`; ``min_context`` - and a declared need already count the prompt and are used as they are. A - task with none of the three is left out. The serving side caps a - ``max_tokens`` need at the native window (:func:`_native_serves`). + ``min_context`` wins when set. Otherwise the stricter of a declared need + (which counts the prompt) and the effective ``max_tokens`` (which does not; + :func:`fits`). A task with none of the three is left out. """ models = validate_named_config_map("models", config.get("models", {})) tasks = validate_named_config_map("tasks", config.get("tasks", {})) @@ -290,18 +307,22 @@ def task_context_needs(config: Mapping[str, Any]) -> dict[str, ContextNeed]: if override is not None: needs[task_name] = ContextNeed(root, override, "min_context") continue - candidates: list[tuple[int, str]] = [] + candidates: list[ContextNeed] = [] if requirements.min_context is not None: - candidates.append((requirements.min_context, "requires")) + candidates.append(ContextNeed(root, requirements.min_context, "requires")) max_tokens = _effective_max_tokens(task_name, task_config, models) if max_tokens is not None: - candidates.append((max_tokens + PROMPT_HEADROOM, "max_tokens")) + candidates.append(ContextNeed(root, max_tokens, "max_tokens")) if candidates: - tokens, source = max(candidates) - needs[task_name] = ContextNeed(root, tokens, source) + needs[task_name] = max(candidates, key=_least_window) return needs +def _least_window(need: ContextNeed) -> int: + """The smallest window that :func:`fits` *need*.""" + return need.tokens + 1 if need.source == "max_tokens" else need.tokens + + def plan_window(plan: DeploymentPlan | Mapping[str, Any]) -> int | None: """The context window *plan* launches its engine with: the smallest across roles, since a prompt overflows on the first that refuses it. ``None`` when @@ -342,44 +363,90 @@ def check_context_fits( config: Mapping[str, Any], infer_plans: Mapping[str, Mapping[str, Any]] | None, tasks: frozenset[str] | None = None, -) -> None: - """Refuse a task whose model cannot serve the window it needs. +) -> list[str]: + """Refuse a task its model's window does not :func:`fit `; return a + warning for each ``max_tokens`` task left less than + :data:`PROMPT_HEADROOM` tokens for its prompt. A model ``sieval run`` serves is checked against its plan; an endpoint against its root model's ``context_limit``. A model whose window is unknown is not checked: an overflow then shows per sample, as before. *tasks* limits the check to the tasks a session runs. """ + shortfalls, warnings = _context_fits(config, infer_plans, tasks) + if shortfalls: + raise ServingShortfallError( + "Tasks need a longer context window than their model provides:\n" + + "\n".join(shortfalls) + ) + return warnings + + +def prompt_room_warnings( + config: Mapping[str, Any], + infer_plans: Mapping[str, Mapping[str, Any]] | None, + tasks: frozenset[str] | None = None, + *, + launched: bool | None = None, +) -> list[str]: + """The warnings :func:`check_context_fits` returns, with no refusal: + *launched* ``True`` for the models *infer_plans* serves only, ``False`` for + the endpoints only, ``None`` for both.""" + return _context_fits(config, infer_plans, tasks, launched)[1] + + +def _context_fits( + config: Mapping[str, Any], + infer_plans: Mapping[str, Mapping[str, Any]] | None, + tasks: frozenset[str] | None, + launched: bool | None = None, +) -> tuple[list[str], list[str]]: + """The shortfalls and the prompt-room warnings of :func:`check_context_fits`.""" models = validate_named_config_map("models", config.get("models", {})) shortfalls: list[str] = [] + warnings: list[str] = [] native_windows: dict[str, int | None] = {} for task_name, need in sorted(task_context_needs(config).items()): if tasks is not None and task_name not in tasks: continue root_config = models.get(need.root, {}) - declared = parse_model_context(need.root, root_config) plan = (infer_plans or {}).get(need.root) - native: int | None = None + if launched is not None and launched != (plan is not None): + continue if plan is not None: if need.root not in native_windows: + declared = parse_model_context(need.root, root_config) native_windows[need.root] = declared.native or _checkpoint_window( plan.get("checkpoint") ) - native = native_windows[need.root] - window = served_window(plan, native) + window = served_window(plan, native_windows[need.root]) what = "serves" else: window = _positive_int( root_config.get("context_limit"), f"Model '{need.root}' context_limit" ) what = "declares context_limit" - if window is not None and not _fits(need, window, native): - shortfalls.append(_shortfall(task_name, need, f"{what} {window}", native)) - if shortfalls: - raise ServingShortfallError( - "Tasks need a longer context window than their model provides:\n" - + "\n".join(shortfalls) - ) + if window is None: + continue + if not fits(need, window): + shortfalls.append(_shortfall(task_name, need, window, f"{what} {window}")) + continue + warning = _prompt_room_warning(task_name, need, window) + if warning is not None: + warnings.append(warning) + return shortfalls, warnings + + +def _prompt_room_warning(task: str, need: ContextNeed, window: int) -> str | None: + """A warning when *need* is a ``max_tokens`` that leaves the prompt less + than :data:`PROMPT_HEADROOM` tokens of *window*.""" + room = window - need.tokens + if need.source != "max_tokens" or room >= PROMPT_HEADROOM: + return None + return ( + f"task '{task}' leaves {room} tokens for the prompt at window {window}; " + "longer prompts overflow per sample" + ) def plan_serving_groups( @@ -389,11 +456,13 @@ def plan_serving_groups( ) -> list[ServingGroup]: """Group the tasks by the plan that serves them, the default group first. - *base_plans* are the recipe plans by root model. A need within the native - window raises that plan, a ``max_tokens`` below native to the native - window itself; one within ``context.max`` gets the group - ``@`` with ``context.scaling``. A plan that serves a task - scoring input then reserves memory for its logits + *base_plans* are the recipe plans by root model. Every window chosen is + declared, never sized to a need (:func:`fits` decides): a recipe window + that fits every task stays; else the native window, when it fits the + task; else ``context.max``, in the group ``@`` with + ``context.scaling`` (or unscaled with ``context.extrapolate``). A window + the user pinned is never raised. A plan that serves a task scoring input + then reserves memory for its logits (:func:`reserve_scoring_headroom`, sized for *gpu_memory_mib* per GPU). Raises :class:`ServingShortfallError` naming every task no plan can serve. """ @@ -435,30 +504,31 @@ def with_headroom( native = context.native or _checkpoint_window(plan.checkpoint) window = served_window(plan, native) pinned = _explicit_window(root, models.get(root, {}), plan.backend) - raise_to: int | None = None + raise_to_native = False scaled: list[str] = [] for task_name in served[root]: need = needs.get(task_name) - if need is None or window is None or _fits(need, window, native): + if need is None or window is None or fits(need, window): continue if pinned is not None: - limit = f"pins {pinned[0]}: {pinned[1]}" + checked, limit = pinned[1], f"pins {pinned[0]}: {pinned[1]}" elif native is None: + checked = window limit = ( f"serves {window} and has no known native window " "(set context.native)" ) - elif _native_serves(need, native): - raise_to = native - continue - elif need.tokens <= native: - raise_to = max(raise_to or 0, need.tokens) + elif fits(need, native): + raise_to_native = True continue elif context.max is None: + checked = native limit = f"supports at most its native {native} (no context.max)" - elif need.tokens > context.max: + elif not fits(need, context.max): + checked = context.max limit = f"supports at most {context.max} (context.max)" elif context.extrapolate and plan.backend not in EXTRAPOLATE_BACKENDS: + checked = native limit = ( f"is served by {plan.backend}, and context.extrapolate " "is supported on sglang only" @@ -469,6 +539,7 @@ def with_headroom( if plan.backend in EXTRAPOLATE_BACKENDS else "" ) + checked = native limit = ( f"needs scaling beyond its native {native}, but context " f"has no scaling parameters{hint}" @@ -476,9 +547,11 @@ def with_headroom( else: scaled.append(task_name) continue - shortfalls.append(_shortfall(task_name, need, limit, native)) + shortfalls.append(_shortfall(task_name, need, checked, limit)) - default_plan = plan if raise_to is None else with_window(plan, raise_to, {}) + default_plan = plan + if raise_to_native and native is not None: + default_plan = with_window(plan, native, {}) default_window = served_window(default_plan, native) stays = [task for task in served[root] if task not in scaled] default_plan, headroom = with_headroom(root, default_plan, stays) @@ -496,15 +569,14 @@ def with_headroom( ) for task_name in stays: windows[task_name] = stay_served - if scaled: - scaled_window = max(needs[task].tokens for task in scaled) - key = f"{root}@{scaled_window}" + if scaled and context.max is not None: + key = f"{root}@{context.max}" scaled_plan, headroom = with_headroom( - root, with_window(plan, scaled_window, context.scaling), scaled + root, with_window(plan, context.max, context.scaling), scaled ) scaled_served = ServedWindow( key, - scaled_window, + context.max, "extrapolate" if context.extrapolate else "scaling", native, context.max, @@ -651,11 +723,16 @@ def _longest_prompt( tasks: Iterable[str], needs: Mapping[str, ContextNeed], window: int | None ) -> int | None: """The longest prompt *tasks* may score, as far as it is known: each one's - context need, at most *window*; ``None`` when a task's is unknown.""" + context need at most *window*, or what a ``max_tokens`` leaves of *window*; + ``None`` when a task's is unknown.""" bounds: list[int] = [] for task in tasks: need = needs.get(task) - known = [v for v in (need.tokens if need else None, window) if v is not None] + if need is not None and need.source == "max_tokens": + known = [window - need.tokens] if window is not None else [] + else: + tokens = need.tokens if need is not None else None + known = [v for v in (tokens, window) if v is not None] if not known: return None bounds.append(min(known)) @@ -702,47 +779,21 @@ def with_window( ) -def _native_serves(need: ContextNeed, native: int | None) -> bool: - """Whether *native* serves *need* unscaled: a need from a ``max_tokens`` - below the native window. Such a task is never scaled, since scaling changes - outputs; its window is raised to *native*, and the prompt gets the - ``native - max_tokens`` tokens left. A longer prompt overflows per sample - (``context_overflow``).""" - return ( - need.source == "max_tokens" - and native is not None - and need.tokens - PROMPT_HEADROOM < native - ) - - -def _required(need: ContextNeed, native: int | None) -> int: - """The smallest window that serves *need*: its tokens, capped at *native* - when the native window serves it.""" - if native is not None and _native_serves(need, native): - return min(need.tokens, native) - return need.tokens - - -def _fits(need: ContextNeed, window: int, native: int | None) -> bool: - return _required(need, native) <= window - - -def _shortfall( - task: str, need: ContextNeed, limit: str, native: int | None = None -) -> str: - tokens = _required(need, native) - source = _SOURCES[need.source] - if tokens != need.tokens: - source = "its max_tokens, served at the native window" - return ( - f"- task '{task}' needs {tokens} tokens ({source}); model '{need.root}' {limit}" - ) +def _shortfall(task: str, need: ContextNeed, window: int, limit: str) -> str: + """One refused task: what :func:`fits` checked at *window*, the largest + the model could serve it at, and *limit*, why no larger one.""" + if need.source == "max_tokens": + checked = ( + f"max_tokens {need.tokens} leaves no room for the prompt at window {window}" + ) + else: + checked = f"needs {need.tokens} tokens ({_SOURCES[need.source]})" + return f"- task '{task}' {checked}; model '{need.root}' {limit}" _SOURCES: dict[str, str] = { "min_context": "its min_context", "requires": "declared by the task", - "max_tokens": f"its max_tokens plus {PROMPT_HEADROOM} for the prompt", } diff --git a/sieval/cli/session.py b/sieval/cli/session.py index 66f302a6..48696f1b 100644 --- a/sieval/cli/session.py +++ b/sieval/cli/session.py @@ -3605,7 +3605,10 @@ def replace_inline( self._validate_external_runtime_obligations(result) for diagnostic in result.diagnostics: logger.warning(self._format_reconcile_diagnostic(diagnostic)) - check_context_fits(self.config, self._infer_plans, self._serving.tasks) + for warning in check_context_fits( + self.config, self._infer_plans, self._serving.tasks + ): + logger.warning(warning) self._task_model_requirements = tuple(records) self._aggregated_requirements = aggregated diff --git a/sieval/cli/validation.py b/sieval/cli/validation.py index d1d0ebe2..8625a904 100644 --- a/sieval/cli/validation.py +++ b/sieval/cli/validation.py @@ -35,7 +35,11 @@ validate_configured_engine_id, validate_task_config_args, ) -from sieval.cli.serving import SessionServing, parse_model_context +from sieval.cli.serving import ( + SessionServing, + parse_model_context, + prompt_room_warnings, +) from sieval.cli.session import EvalSession, RootConfigDict from sieval.core.models import check_extra from sieval.core.models.capabilities import ( @@ -1058,6 +1062,14 @@ def _capability_reconcile_check( for diagnostic in result.diagnostics if diagnostic.severity.value == "warning" ] + # An endpoint's prompt room; a launched engine's is on `sieval run + # --dry-run`'s serving check, once per group. + warnings += prompt_room_warnings( + session.config, + infer_plans, + serving.tasks if serving is not None else None, + launched=False, + ) if warnings: check["warnings"] = warnings return [check, _dependencies_check(session)] diff --git a/tests/unit/cli/test_run.py b/tests/unit/cli/test_run.py index 9c653817..fff7bfff 100644 --- a/tests/unit/cli/test_run.py +++ b/tests/unit/cli/test_run.py @@ -2032,7 +2032,7 @@ def _config(tmp_path: Path) -> Path: "path": "/tmp/ckpt", "context": { "native": 32768, - "max": 131072, + "max": 100000, "scaling": {"json_model_override_args": "{}"}, }, } @@ -2287,7 +2287,7 @@ def test_only_a_scaled_sglang_engine_may_exceed_its_derived_window( "path": "/tmp/ckpt", "context": { "native": 32768, - "max": 131072, + "max": 100000, "scaling": {"json_model_override_args": "{}"}, }, **({"infer": {"env": user_env}} if user_env else {}), @@ -2365,15 +2365,15 @@ def test_an_extrapolated_engine_gets_the_window_and_no_scaling(self) -> None: config, {"m": unwrap_proxies(plan)}, {"m": (plan, base)} ) - assert extrapolated.key == "m@52096" + assert extrapolated.key == "m@65536" params = extrapolated.prepared["m"][0].assignments[0].engine_params - assert params["context_length"] == 52096 + assert params["context_length"] == 65536 assert "json_model_override_args" not in params (command,) = extrapolated.prepared["m"][1] assert command.env["SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN"] == "1" assert extrapolated.serving().windows == { "long": { - "window": 52096, + "window": 65536, "method": "extrapolate", "native": 32768, "max": 65536, @@ -2496,7 +2496,10 @@ async def test_dry_run_fails_the_serving_check_on_a_shortfall( (check,) = cast(list, result["checks"]) assert check["name"] == "serving" assert check["ok"] is False - assert "task 'long' needs 100000 tokens" in check["detail"] + assert ( + "task 'long' max_tokens 95904 leaves no room for the prompt at window 32768" + in check["detail"] + ) assert result["n_errors"] == 1 assert (result["tasks"], result["serving"]) == (None, None) @@ -2606,6 +2609,88 @@ async def test_roomy_plan_adds_no_serving_check(self, tmp_path: Path) -> None: (fit,) = cast(dict, result["serving"])["default"]["engines"]["m"]["memory"] assert (fit["tp"], fit["tp_source"], fit["low_room"]) == (4, "stated", False) + @staticmethod + def _scaled_config(tmp_path: Path, **overrides: object) -> Path: + """A task past native 40960: its group serves context.max (#289).""" + path = TestDryRunSizing._config(tmp_path, **overrides) + config = yaml.safe_load(path.read_text()) + model = config["models"]["m"] + del model["infer"]["overrides"]["context_length"] + model["context"] = { + "native": 40960, + "max": 131072, + "scaling": {"json_model_override_args": "{}"}, + } + config["datasets"] = {"d": {"class": "GSM8KDataset"}} + config["tasks"] = { + "long": { + "class": "GSM8KZeroShotGenTask", + "dataset": "d", + "infer_args": {"max_tokens": 60000}, + } + } + path.write_text(yaml.safe_dump(config)) + return path + + @pytest.mark.anyio + @pytest.mark.parametrize( + ("overrides", "gpu", "tp", "source"), + [ + ({}, "NVIDIA H100:1:81559", 1, "sized"), + ({"tp_size": 2}, "NVIDIA H100:2:81559", 2, "stated"), + ], + ids=["sized", "stated"], + ) + async def test_a_scaled_engine_is_checked_at_context_max( + self, + tmp_path: Path, + overrides: dict[str, object], + gpu: str, + tp: int, + source: str, + ) -> None: + """The scaled engine launches at context.max with the recipe's tp, + sized or stated, and its memory is checked at that window.""" + result = await self._dry_run(self._scaled_config(tmp_path, **overrides), gpu) + + serving = cast(dict, result["serving"]) + assert list(serving) == ["default", "m@131072"] + engine = serving["m@131072"]["engines"]["m@131072"] + assert engine["window"] == 131072 + (fit,) = engine["memory"] + assert (fit["tp"], fit["tp_source"], fit["context"]) == (tp, source, 131072) + # One full 131072 sequence fits beside the weights on one H100, with + # little room: the serving check warns, naming the scaled engine. + assert (fit["fits"], fit["low_room"]) == (True, True) + checks = cast(list, result["checks"]) + warnings = next(c for c in checks if c["name"] == "serving")["warnings"] + assert any( + w.startswith("engine m@131072:") and "context 131072" in w for w in warnings + ) + + @pytest.mark.anyio + async def test_little_prompt_room_warns_on_the_serving_check( + self, tmp_path: Path + ) -> None: + """#289: native 40960 serves max_tokens 38000 with 2960 tokens left.""" + path = self._scaled_config(tmp_path, tp_size=4) + config = yaml.safe_load(path.read_text()) + config["tasks"]["long"]["infer_args"]["max_tokens"] = 38000 + path.write_text(yaml.safe_dump(config)) + + result = await self._dry_run(path, "NVIDIA H100:4:81559") + + assert list(cast(dict, result["serving"])) == ["default"] + serving = next( + c for c in cast(list, result["checks"]) if c["name"] == "serving" + ) + assert serving["ok"] is True + assert serving["warnings"] == [ + "task 'long' leaves 2960 tokens for the prompt at window 40960; " + "longer prompts overflow per sample" + ] + assert result["n_warnings"] == 1 + @pytest.mark.anyio async def test_memory_unknown_without_the_checkpoint(self, tmp_path: Path) -> None: from sieval.cli.run import _serving_json diff --git a/tests/unit/cli/test_serving.py b/tests/unit/cli/test_serving.py index 35558f3d..e903a8f3 100644 --- a/tests/unit/cli/test_serving.py +++ b/tests/unit/cli/test_serving.py @@ -22,6 +22,7 @@ parse_model_context, plan_serving_groups, plan_window, + prompt_room_warnings, reserve_scoring_headroom, served_contexts, task_context_needs, @@ -104,15 +105,15 @@ def _gen(max_tokens: int) -> dict[str, Any]: class TestTaskContextNeeds: def test_need_is_max_tokens_from_infer_args_then_model_args(self) -> None: - """Plus PROMPT_HEADROOM: a window of exactly max_tokens fits no prompt.""" + """The raw max_tokens: :func:`fits` leaves the prompt the rest.""" config = _config( {"short": {}, "long": _gen(38000)}, model={"path": "/models/m", "args": {"max_tokens": 4096}}, ) assert task_context_needs(config) == { - "short": ContextNeed("m", 8192, "max_tokens"), - "long": ContextNeed("m", 42096, "max_tokens"), + "short": ContextNeed("m", 4096, "max_tokens"), + "long": ContextNeed("m", 38000, "max_tokens"), } def test_derived_model_inherits_max_tokens_from_its_base(self) -> None: @@ -120,12 +121,13 @@ def test_derived_model_inherits_max_tokens_from_its_base(self) -> None: config["models"]["m"]["args"] = {"max_completion_tokens": 2048} config["models"]["child"] = {"base": "m"} - assert task_context_needs(config) == {"t": ContextNeed("m", 6144, "max_tokens")} + assert task_context_needs(config) == {"t": ContextNeed("m", 2048, "max_tokens")} def test_declared_need_wins_when_larger_and_override_wins_always(self) -> None: config = _config( { "declared": {"class": LONG, **_gen(1024)}, + "equal": {"class": LONG, **_gen(50000)}, "max_tokens": {"class": LONG, **_gen(60000)}, "override": {"class": LONG, "min_context": 8192}, } @@ -133,7 +135,10 @@ def test_declared_need_wins_when_larger_and_override_wins_always(self) -> None: assert task_context_needs(config) == { "declared": ContextNeed("m", 50000, "requires"), - "max_tokens": ContextNeed("m", 64096, "max_tokens"), + # A window of 50000 fits the declared need but leaves max_tokens + # 50000 no prompt. + "equal": ContextNeed("m", 50000, "max_tokens"), + "max_tokens": ContextNeed("m", 60000, "max_tokens"), "override": ContextNeed("m", 8192, "min_context"), } @@ -252,8 +257,9 @@ def test_max_tokens_below_native_raises_to_native(self) -> None: assert plan_window(group.plans["m"]) == 40960 - def test_max_tokens_within_headroom_of_native_is_never_scaled(self) -> None: - """38000 + headroom exceeds native 40960, but scaling changes outputs.""" + def test_max_tokens_below_native_is_never_scaled(self) -> None: + """Native 40960 leaves 2960 tokens for the prompt; scaling would change + outputs, so a longer prompt overflows per sample instead.""" config = _config( {"t": _gen(38000)}, model={ @@ -270,9 +276,11 @@ def test_max_tokens_within_headroom_of_native_is_never_scaled(self) -> None: DEFAULT_GROUP, 40960, "native", 40960, 131072 ) - def test_max_tokens_at_or_above_native_gets_headroom_and_scales(self) -> None: + def test_max_tokens_at_or_above_native_scales_to_max(self) -> None: + """At native, max_tokens leaves no prompt: the task is scaled, and the + scaled window is context.max, not the largest need.""" config = _config( - {"at": _gen(40960), "above": _gen(60000)}, + {"below": _gen(40959), "at": _gen(40960), "above": _gen(60000)}, model={ "path": "/models/m", "context": {"native": 40960, "max": 131072, "scaling": {"x": 1}}, @@ -281,16 +289,42 @@ def test_max_tokens_at_or_above_native_gets_headroom_and_scales(self) -> None: default, scaled = plan_serving_groups(config, {"m": _plan(32768)}) - assert default.tasks == () - assert scaled.key == "m@64096" + assert default.tasks == ("below",) + assert plan_window(default.plans["m"]) == 40960 + assert scaled.key == "m@131072" assert scaled.tasks == ("at", "above") assert scaled.windows["at"] == ServedWindow( - "m@64096", 64096, "scaling", 40960, 131072 + "m@131072", 131072, "scaling", 40960, 131072 ) - def test_pinned_window_below_native_refuses_at_the_native_need(self) -> None: + @pytest.mark.parametrize( + "tasks", + [{"t": _gen(44096 - 4096)}, {"t": _gen(42096 - 4096), "u": _gen(42000)}], + ids=["need-44096", "need-42096"], + ) + def test_scaled_window_is_the_same_for_every_task_subset( + self, tasks: dict[str, dict[str, Any]] + ) -> None: + """Before #289 these launched at 44096 and 42096, two inference + identities for one model and profile.""" config = _config( - {"t": _gen(38000)}, + tasks, + model={ + "path": "/models/m", + "context": {"native": 32768, "max": 131072, "scaling": {"x": 1}}, + }, + ) + + _, scaled = plan_serving_groups(config, {"m": _plan(32768)}) + + assert scaled.key == "m@131072" + assert plan_window(scaled.plans["m"]) == 131072 + + @pytest.mark.parametrize(("max_tokens", "fits"), [(38999, True), (39000, False)]) + def test_pinned_window_is_never_raised(self, max_tokens: int, fits: bool) -> None: + """Below the native 40960, but pinned: a task fits it or is refused.""" + config = _config( + {"t": _gen(max_tokens)}, model={ "path": "/models/m", "context": {"native": 40960}, @@ -298,27 +332,33 @@ def test_pinned_window_below_native_refuses_at_the_native_need(self) -> None: }, ) + if fits: + (group,) = plan_serving_groups(config, {"m": _plan(39000)}) + assert plan_window(group.plans["m"]) == 39000 + return with pytest.raises(ServingShortfallError) as caught: plan_serving_groups(config, {"m": _plan(39000)}) - assert ( - "task 't' needs 40960 tokens (its max_tokens, served at the native " - "window); model 'm' pins infer.overrides.context_length: 39000" + "task 't' max_tokens 39000 leaves no room for the prompt at window " + "39000; model 'm' pins infer.overrides.context_length: 39000" in str(caught.value) ) @pytest.mark.parametrize( ("task", "window", "scaled"), [ - ({"min_context": 36000, **_gen(30000)}, 36000, False), - ({"class": LONG}, 50000, True), + ({"min_context": 36000, **_gen(30000)}, 40960, False), + ({"min_context": 40960}, 40960, False), + ({"min_context": 40961}, 131072, True), + ({"class": LONG}, 131072, True), ], - ids=["min_context-override", "declared-requires"], + ids=["min_context-override", "at-native", "past-native", "declared-requires"], ) - def test_explicit_min_context_is_used_unchanged( + def test_min_context_is_a_hard_need_served_at_a_declared_window( self, task: dict[str, Any], window: int, scaled: bool ) -> None: - """It already counts the prompt: no headroom, no rounding up to native.""" + """It counts the prompt, so it may fill the window; the window is + native or context.max, never the need.""" config = _config( {"t": task}, model={ @@ -333,18 +373,25 @@ def test_explicit_min_context_is_used_unchanged( assert (group.windows["t"].method != "native") is scaled assert plan_window(group.plans["m"]) == window - def test_unknown_native_keeps_max_tokens_plus_headroom(self) -> None: + @pytest.mark.parametrize(("max_tokens", "fits"), [(40959, True), (40960, False)]) + def test_unknown_native_checks_the_recipe_window( + self, max_tokens: int, fits: bool + ) -> None: config = _config( - {"t": _gen(38000)}, + {"t": _gen(max_tokens)}, model={"path": "/models/m", "context": {"max": 131072, "scaling": {}}}, ) + if fits: + (group,) = plan_serving_groups(config, {"m": _plan(40960)}) + assert plan_window(group.plans["m"]) == 40960 + return with pytest.raises(ServingShortfallError) as caught: plan_serving_groups(config, {"m": _plan(40960)}) - assert ( - "task 't' needs 42096 tokens (its max_tokens plus 4096 for the prompt); " - "model 'm' serves 40960 and has no known native window" in str(caught.value) + "task 't' max_tokens 40960 leaves no room for the prompt at window " + "40960; model 'm' serves 40960 and has no known native window" + in str(caught.value) ) def test_native_window_defaults_to_the_checkpoint_config( @@ -375,13 +422,13 @@ def test_need_beyond_native_gets_a_scaled_group_of_its_own(self) -> None: assert default.tasks == ("short",) assert plan_window(default.plans["m"]) == 32768 - assert scaled.key == "m@124096" + assert scaled.key == "m@131072" assert scaled.tasks == ("long", "longer") params = scaled.plans["m"].assignments[0].engine_params - assert params["context_length"] == 124096 + assert params["context_length"] == 131072 assert params["json_model_override_args"] == scaling["json_model_override_args"] assert scaled.windows["long"] == ServedWindow( - "m@124096", 124096, "scaling", 32768, 131072 + "m@131072", 131072, "scaling", 32768, 131072 ) assert (default.launches, scaled.launches) == ({"m"}, {"m"}) @@ -398,11 +445,11 @@ def test_extrapolate_serves_beyond_native_unscaled(self) -> None: default, extrapolated = plan_serving_groups(config, {"m": _plan(32768)}) assert default.tasks == ("short",) - assert extrapolated.key == "m@52096" + assert extrapolated.key == "m@65536" params = extrapolated.plans["m"].assignments[0].engine_params - assert params == {"context_length": 52096} + assert params == {"context_length": 65536} assert extrapolated.windows["long"].record() == { - "window": 52096, + "window": 65536, "method": "extrapolate", "native": 32768, "max": 65536, @@ -533,30 +580,33 @@ def test_malformed_window_override_names_the_model(self) -> None: plan_serving_groups(config, {"m": _plan(32768)}) @pytest.mark.parametrize( - ("model", "message"), + ("model", "window", "message"), [ - ({"context": {"native": 32768}}, "at most its native 32768"), - ({"context": {"native": 32768, "max": 65536}}, "at most 65536"), + ({"context": {"native": 32768}}, 32768, "at most its native 32768"), + ({"context": {"native": 32768, "max": 65536}}, 65536, "at most 65536"), ( {"context": {"native": 32768, "max": 131072}}, + 32768, "no scaling parameters", ), - ({}, "no known native window"), + ({}, 32768, "no known native window"), ( { "context": {"native": 131072}, "infer": {"overrides": {"context_length": 32768}}, }, + 32768, "pins infer.overrides.context_length: 32768", ), ( {"context": {"native": 131072}, "context_limit": 32768}, + 32768, "pins context_limit: 32768", ), ], ) def test_unservable_need_is_refused_naming_task_and_shortfall( - self, model: dict[str, Any], message: str + self, model: dict[str, Any], window: int, message: str ) -> None: config = _config( {"ok": _gen(1024), "long": _gen(100000)}, @@ -568,12 +618,26 @@ def test_unservable_need_is_refused_naming_task_and_shortfall( text = str(caught.value) assert ( - "task 'long' needs 104096 tokens " - "(its max_tokens plus 4096 for the prompt)" in text + f"- task 'long' max_tokens 100000 leaves no room for the prompt at " + f"window {window}; model 'm' " in text ) assert message in text assert "'ok'" not in text + def test_a_min_context_shortfall_names_the_need(self) -> None: + config = _config( + {"t": {"min_context": 40961}}, + model={"path": "/models/m", "context": {"native": 40960}}, + ) + + with pytest.raises(ServingShortfallError) as caught: + plan_serving_groups(config, {"m": _plan(32768)}) + + assert ( + "- task 't' needs 40961 tokens (its min_context); model 'm' supports " + "at most its native 40960 (no context.max)" in str(caught.value) + ) + def test_external_model_tasks_stay_in_the_default_group(self) -> None: config = _config({"api_task": {"model": "api"}, "local": {"model": "m"}}) config["models"]["api"] = {"name": "x", "api_base": "http://x/v1"} @@ -595,70 +659,125 @@ def test_grouping_is_stable_across_task_order(self) -> None: [g.key for g in plan_serving_groups(cfg, {"m": _plan()})] for cfg in (forward, backward) ] - assert keys[0] == keys[1] == [DEFAULT_GROUP, "m@94096"] + assert keys[0] == keys[1] == [DEFAULT_GROUP, "m@131072"] + + +def _served(window: int) -> dict[str, Any]: + """A launched plan as ``infer_plans.yaml`` records it.""" + return { + "backend": "sglang", + "checkpoint": "/models/m", + "assignments": [{"engine_params": {"context_length": window}}], + } + + +def _endpoint(**model: Any) -> dict[str, Any]: + return {"name": "x", "api_base": "http://x/v1", **model} class TestCheckContextFits: def test_served_model_is_checked_against_its_plan(self) -> None: config = _config({"long": _gen(38000)}) - plans = { - "m": { - "backend": "sglang", - "checkpoint": "/models/m", - "assignments": [{"engine_params": {"context_length": 32768}}], - } - } + plans = {"m": _served(32768)} with pytest.raises(ServingShortfallError, match="serves 32768"): check_context_fits(config, plans) - check_context_fits(config, plans, frozenset({"other"})) - - def test_served_native_window_fits_a_max_tokens_below_it(self) -> None: - plan = { - "backend": "sglang", - "checkpoint": "/models/m", - "assignments": [{"engine_params": {"context_length": 40960}}], - } + assert check_context_fits(config, plans, frozenset({"other"})) == [] + + def test_one_window_one_verdict_whoever_serves_it(self) -> None: + """#289: max_tokens 38000 on Qwen3-30B-A3B (native 40960). A launched + engine at native, an endpoint declaring the same window and an + endpoint declaring none all run it; the two known windows warn.""" + warning = ( + "task 't' leaves 2960 tokens for the prompt at window 40960; " + "longer prompts overflow per sample" + ) native = {"path": "/models/m", "context": {"native": 40960}} + launched = _config({"t": _gen(38000)}, model=native) + (group,) = plan_serving_groups(launched, {"m": _plan(32768)}) + assert plan_window(group.plans["m"]) == 40960 - check_context_fits(_config({"t": _gen(38000)}, model=native), {"m": plan}) - with pytest.raises(ServingShortfallError) as caught: - check_context_fits( - _config({"t": _gen(38000)}, model={"path": "/models/m"}), {"m": plan} - ) - assert "needs 42096 tokens" in str(caught.value) + assert check_context_fits(launched, {"m": _served(40960)}) == [warning] + assert check_context_fits( + _config({"t": _gen(38000)}, model=_endpoint(context_limit=40960)), None + ) == [warning] + assert ( + check_context_fits(_config({"t": _gen(38000)}, model=_endpoint()), None) + == [] + ) def test_external_model_is_checked_against_its_context_limit(self) -> None: - config = _config( - {"long": _gen(38000)}, - model={"name": "x", "api_base": "http://x/v1", "context_limit": 32768}, - ) + config = _config({"long": _gen(38000)}, model=_endpoint(context_limit=32768)) - with pytest.raises(ServingShortfallError, match="declares context_limit 32768"): + with pytest.raises( + ServingShortfallError, + match="max_tokens 38000 leaves no room for the prompt at window 32768; " + "model 'm' declares context_limit 32768", + ): check_context_fits(config, None) - @pytest.mark.parametrize(("max_tokens", "fits"), [(28672, True), (28673, False)]) - def test_context_limit_leaves_prompt_headroom( - self, max_tokens: int, fits: bool + @pytest.mark.parametrize( + ("max_tokens", "room"), + [(28672, None), (28673, 4095), (32767, 1), (32768, "refused")], + ) + def test_context_limit_refuses_only_max_tokens_with_no_prompt_room( + self, max_tokens: int, room: int | str | None + ) -> None: + """Below PROMPT_HEADROOM of room warns; no room at all refuses.""" + config = _config({"t": _gen(max_tokens)}, model=_endpoint(context_limit=32768)) + + if room == "refused": + with pytest.raises( + ServingShortfallError, + match="max_tokens 32768 leaves no room for the prompt at window 32768", + ): + check_context_fits(config, None) + return + warnings = check_context_fits(config, None) + if room is None: + assert warnings == [] + else: + assert warnings == [ + f"task 't' leaves {room} tokens for the prompt at window 32768; " + "longer prompts overflow per sample" + ] + + @pytest.mark.parametrize(("min_context", "fits"), [(32768, True), (32769, False)]) + def test_min_context_is_unchanged_and_never_warns( + self, min_context: int, fits: bool ) -> None: config = _config( - {"t": _gen(max_tokens)}, - model={"name": "x", "api_base": "http://x/v1", "context_limit": 32768}, + {"t": {"min_context": min_context, **_gen(32000)}}, + model=_endpoint(context_limit=32768), ) if fits: - check_context_fits(config, None) + assert check_context_fits(config, None) == [] else: with pytest.raises(ServingShortfallError, match="needs 32769 tokens"): check_context_fits(config, None) + def test_warnings_split_by_who_serves_the_model(self) -> None: + """The dry-run reports a launched engine's on the serving check, an + endpoint's on capability_reconcile: each once.""" + config = _config({"t": {**_gen(38000), "model": "m"}}) + config["models"]["api"] = _endpoint(context_limit=40960) + config["tasks"]["u"] = {**config["tasks"]["t"], "model": "api"} + + plans = {"m": _served(40960)} + + def tasks(launched: bool | None) -> list[str]: + warnings = prompt_room_warnings(config, plans, launched=launched) + return [w.split("'")[1] for w in warnings] + + assert tasks(False) == ["u"] + assert tasks(True) == ["t"] + assert tasks(None) == ["t", "u"] + def test_unknown_window_is_not_checked(self) -> None: - config = _config( - {"long": _gen(1_000_000)}, - model={"name": "x", "api_base": "http://x/v1"}, - ) + config = _config({"long": _gen(1_000_000)}, model=_endpoint()) - check_context_fits(config, None) + assert check_context_fits(config, None) == [] def _checkpoint(tmp_path: Path, **config: Any) -> str: @@ -938,7 +1057,7 @@ def test_a_scaled_group_sizes_its_own_headroom(self, tmp_path: Path) -> None: assert scaled.tasks == ("long_ppl",) assert scaled.headroom["m"].tasks == ("long_ppl",) assert _params(scaled.plans["m"])["chunked_prefill_size"] == 3072 - assert _params(scaled.plans["m"])["context_length"] == 60000 + assert _params(scaled.plans["m"])["context_length"] == 131072 def test_the_longest_scored_prompt_is_the_need_within_the_window( self, tmp_path: Path @@ -954,6 +1073,22 @@ def test_the_longest_scored_prompt_is_the_need_within_the_window( assert group.headroom["m"].prefill_tokens == 2048 + @pytest.mark.parametrize(("max_tokens", "prefill"), [(30720, 2048), (1024, 3072)]) + def test_a_max_tokens_bounds_the_scored_prompt_by_the_room_it_leaves( + self, tmp_path: Path, max_tokens: int, prefill: int + ) -> None: + """32768 - 30720 leaves 2048; 31744 left is above the GPU budget.""" + checkpoint = _checkpoint(tmp_path) + plan = _plan(checkpoint=checkpoint, mem_fraction_static=0.87) + config = _config( + {"ppl": {"class": SCORING, **_gen(max_tokens)}}, + model={"path": checkpoint}, + ) + + (group,) = plan_serving_groups(config, {"m": plan}, H100_MIB) + + assert group.headroom["m"].prefill_tokens == prefill + def test_unknown_gpu_memory_reports_the_unchanged_plan( self, tmp_path: Path ) -> None: diff --git a/tests/unit/cli/test_validation.py b/tests/unit/cli/test_validation.py index 0b475d12..2c62ef39 100644 --- a/tests/unit/cli/test_validation.py +++ b/tests/unit/cli/test_validation.py @@ -1855,6 +1855,51 @@ def test_dependencies_check_passes_with_nothing_declared(self, tmp_path): "plan": {"unmet": {}, "declared": {}}, } + @pytest.mark.parametrize( + ("max_tokens", "warnings"), + [ + ( + 38000, + [ + "task 't' leaves 2960 tokens for the prompt at window 40960; " + "longer prompts overflow per sample" + ], + ), + (32768, None), + ], + ) + def test_an_endpoint_with_little_prompt_room_warns( + self, tmp_path: Path, max_tokens: int, warnings: list[str] | None + ) -> None: + """#289: refused before as needing 42096; now the run goes ahead.""" + from sieval.cli.validation import run_dry_run + + config = tmp_path / "endpoint.yaml" + config.write_text( + "models:\n" + " m:\n" + " api_base: http://x/v1\n" + " name: x\n" + " context_limit: 40960\n" + "datasets:\n" + " d:\n" + " class: sieval.datasets.aime_2024.AIME2024Dataset\n" + "tasks:\n" + " t:\n" + f" class: {self._AIME_TASK}\n" + " dataset: d\n" + " model: m\n" + f" infer_args: {{max_tokens: {max_tokens}}}\n", + encoding="utf-8", + ) + + result = run_dry_run(config) + + check = next(c for c in result["checks"] if c["name"] == "capability_reconcile") + assert check["ok"] is True + assert check.get("warnings") == warnings + assert result["n_errors"] == 0 + def test_dependencies_check_fails_on_an_unmet_declaration(self, tmp_path): from sieval.cli.validation import run_dry_run from sieval.tasks.aime_2024_0shot_gen import AIME2024ZeroShotGenTask From ef4107bc746afb9400ed6a56251b3a29e5328329 Mon Sep 17 00:00:00 2001 From: Ethan Date: Tue, 10 Nov 2026 17:40:06 +0800 Subject: [PATCH 2/2] fix(serving): keep the full window for scoring; raise to native on short prompt room Scoring calls send their own max_tokens, so a generation max_tokens from the model's args no longer bounds the longest scored prompt; it had cut the logits reserve (3072 to 1024 prefill tokens at 32768). A recipe window below native that leaves a max_tokens task under PROMPT_HEADROOM of prompt room is raised to native, as before #289; only a pinned window stays and warns. prompt_room_warnings takes launched as a required bool. Docs, CHANGELOG and docstrings trimmed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 +-- docs/guide/configuration.md | 7 ++- docs/guide/infer.md | 58 ++++++++------------ sieval/cli/serving.py | 96 +++++++++++++++++----------------- tests/unit/cli/test_serving.py | 41 +++++++++++---- 5 files changed, 108 insertions(+), 102 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e22def..14ed0b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed -- **One rule decides whether a context window serves a task, whoever picked the window (#289).** A `max_tokens` task fits a window that leaves at least one token for the prompt; `min_context` and a task's declared need still count the prompt and must fit whole. Before, a launched engine at its native window ran `max_tokens: 38000` at 40960, while an endpoint declaring `context_limit: 40960` refused it as needing 42096 (`max_tokens` plus 4096 for the prompt). Both now run it, and both warn, on `--dry-run` and at run time, when the prompt is left under 4096 tokens: `task 't' leaves 2960 tokens for the prompt at window 40960; longer prompts overflow per sample`. A `max_tokens` equal to the window is refused as `max_tokens 32768 leaves no room for the prompt at window 32768`. A longer prompt still overflows per sample (`context_overflow`). -- **A served window is a declared one, never sized to a need.** A scaled group serves `context.max` and is keyed `@`, the same for every subset of tasks; it used to serve, and be keyed by, the largest need among this run's tasks. A `min_context` or declared need between the recipe window and `native` now raises the default group to `native`, not to the need. A recipe window that serves every task still stays, and a window pinned by `infer.overrides` or `context_limit` is still never raised. A scaled engine keeps the recipe's tp; `--dry-run` checks its memory at `context.max` and warns when little KV room is left. Raise tp in `infer.overrides` if one `context.max` sequence does not fit. -- **Resume:** the served window is in `identity.inference`, so a directory recorded at a need-sized window (a scaled group, or a default group raised to a `min_context` below `native`) is refused on `inference` and needs inference again: start fresh, or pin the recorded window with `infer.overrides`. `IDENTITY_SCHEMA` stays 5 and `RESULT_SCHEMA_VERSION` stays 1: no served token changes at a given window, and the window change is already caught by the inference identity. -- `sieval.cli.serving`: `check_context_fits` returns the prompt-room warnings; `task_context_needs` carries a `max_tokens` need as the raw `max_tokens`, no longer plus `PROMPT_HEADROOM`; `fits(need, window)` is the one rule. `PROMPT_HEADROOM` is the warning threshold only. +- **One rule decides whether a context window serves a task, whoever picked it (#289).** `min_context` and a declared need count the prompt (window ≥ need); `max_tokens` counts the output only (window > `max_tokens`). An endpoint with `context_limit: 40960` now runs `max_tokens: 38000`, as a launched engine already did, instead of refusing it as needing 42096. Under 4096 tokens of prompt room is a warning (`task 't' leaves 2960 tokens for the prompt at window 40960; ...`), on `--dry-run` and at run time. +- **A served window is a declared one, never sized to a need.** A scaled group serves and is keyed by `context.max` (`@`) for every task subset. A need between the recipe window and `native`, including a `max_tokens` left under 4096 tokens of prompt room by the recipe window, raises the default group to `native`. A pinned window is never raised. A scaled engine keeps the recipe's tp; `--dry-run` warns on low KV room at `context.max`. +- **Resume:** a directory recorded at a need-sized window (a scaled group, or a default group raised to a `min_context` below `native`) is refused on `inference`: start fresh, or pin the recorded window with `infer.overrides`. `IDENTITY_SCHEMA` and `RESULT_SCHEMA_VERSION` are unchanged; the window is already in `identity.inference`. +- `sieval.cli.serving`: new `fits(need, window)` and `prompt_room_warnings(...)`; `check_context_fits` returns the warnings; a `max_tokens` need is the raw `max_tokens`. ## [0.15.7] - 2026-11-08 diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 6aaefc8e..f5b789c9 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -603,10 +603,9 @@ An unknown mode, or `--mode` without `--dry-run`, exits 2. its engine. `gpu` is what the plans were resolved for. - `serving` has one entry per serving group (`default`, or `@`): its `tasks`, their served windows, the `engines` it launches and whether any - starts. The `serving` check fails on a window shortfall, and warns, as - `task '' leaves tokens for the prompt at window ; ...`, for a - `max_tokens` task with under 4096 tokens of prompt room on a launched - engine; an endpoint's goes to `capability_reconcile`. See + starts. The `serving` check fails on a window shortfall and warns on a + `max_tokens` task left under 4096 tokens of prompt room (an endpoint's + warning goes to `capability_reconcile`). See [Serving plans](infer.md#serving-plans-context-window-and-dialect). An engine serving input scoring has `headroom` ([Input-scoring headroom](infer.md#input-scoring-headroom)). diff --git a/docs/guide/infer.md b/docs/guide/infer.md index 2ca0494d..322069a9 100644 --- a/docs/guide/infer.md +++ b/docs/guide/infer.md @@ -227,20 +227,16 @@ included). A task needs its `min_context` if set, else the stricter of what its class declares (`TaskRequirements.min_context`; RULER: `max_seq_length` plus any -unreserved thinking budget) and its effective `max_tokens`. One rule decides -whether a window serves a task, both when SiEval picks the window and when it -checks one it did not pick: +unreserved thinking budget) and its effective `max_tokens`. One rule picks a +window and checks one SiEval did not pick: -- `min_context` (either form) counts prompt and output: the window must be at - least that. -- `max_tokens` counts the output only: the window must exceed it by at least - one token. Whatever is left is the prompt's room; a longer prompt is that - sample's [context overflow](metrics.md#always-present). Room under 4096 - tokens is warned about, naming the task, the room and the window; the run - goes ahead. +- `min_context` (either form) counts prompt and output: window ≥ need. +- `max_tokens` counts output only: window > `max_tokens`. The rest is the + prompt's room; a longer prompt is that sample's + [context overflow](metrics.md#always-present). Room under 4096 tokens is + warned about, not refused. -A task whose prompts are longer than the room `max_tokens` leaves sets -`min_context`. A root model states what it can serve: +A root model states what it can serve: ```yaml models: @@ -258,42 +254,34 @@ tasks: min_context: 38000 # optional: override the derived need ``` -A served window is always a declared one, never sized to a task's need: +A served window is a declared one, never sized to a need: -- The recipe window serves every task: nothing changes. -- `native` serves a task the recipe window does not: the plan's - `context_length` / `max_model_len` is raised to `native`, for every task on - that model. +- The recipe window serves every task with 4096 tokens of prompt room: + nothing changes. +- Otherwise, when `native` serves the task: the plan's `context_length` / + `max_model_len` is raised to `native`, for every task on that model. - Only `max` serves it: a plan of its own at `max`, with `scaling`, in serving - group `@`. Every subset of tasks gets the same window, so a - resume across task subsets keeps one inference identity. Scaling changes + group `@`, the same for every subset of tasks. Scaling changes short-context outputs, so no other task shares that engine. SGLang launches it with `SGLANG_ALLOW_OVERWRITE_LONGER_CONTEXT_LEN=1`, so `max` is the only - bound: SGLang no longer checks the window against the scaling. `infer.env` - overrides it. The engine keeps the recipe's tp; `--dry-run` reports its - memory at `max` under `serving..engines..memory`, with the - low-KV warning when one `max` sequence leaves little room. Raise tp in - `infer.overrides` if it does not fit. + bound; `infer.env` overrides it. The engine keeps the recipe's tp; + `--dry-run` warns when one `max` sequence leaves little KV room. Raise tp in + `infer.overrides` then. - Within `max` with `extrapolate: true` instead of `scaling`: the same group and variable, with no scaling parameters. SGLang only; exclusive with `scaling`. - Otherwise, or beyond a window pinned by `infer.overrides` / `context_limit` - (never raised): refused before launch, naming the task, what was checked - (`needs 40961 tokens (its min_context)`, or `max_tokens 40960 leaves no room - for the prompt at window 40960`) and the model's limit. An unknown native - window refuses any raise (set `context.native`). + (never raised): refused before launch, naming the task, what was checked and + the model's limit. An unknown native window refuses any raise (set + `context.native`). An `api_base` model is checked against its `context_limit` by the same rule; -without one, an overflow shows per sample. With `max_tokens: 38000` on a model -whose window is 40960, a launched engine and an endpoint declaring -`context_limit: 40960` both run the task and both warn that it leaves 2960 -tokens for the prompt; an endpoint without `context_limit` runs it unchecked. +without one, an overflow shows per sample. **Resume.** The served window is in each task's inference identity, and a resume never reruns inference. A run recorded at a window this release serves -differently (a scaled group recorded at its largest need by an earlier -release, now at `max`) is refused: start fresh, or pin the recorded window -with `infer.overrides`. +differently is refused: start fresh, or pin the recorded window with +`infer.overrides`. Groups run one after another, each group's engines stopped before the next start; an engine starts only for a group with tasks on it. `--serving-group diff --git a/sieval/cli/serving.py b/sieval/cli/serving.py index 821c5044..e33b937f 100644 --- a/sieval/cli/serving.py +++ b/sieval/cli/serving.py @@ -41,10 +41,8 @@ _MAX_TOKENS_KEYS = ("max_tokens", "max_completion_tokens") -#: The prompt room below which a ``max_tokens`` task is warned about: engines -#: refuse prompt + ``max_tokens`` > window, so a longer prompt overflows per -#: sample (``context_overflow``). It sizes no window and refuses no run -#: (:func:`fits`); a task whose prompts are longer sets ``min_context``. +#: Prompt room under which a ``max_tokens`` task is warned about, and a +#: recipe window below native is raised to native. It refuses no run. PROMPT_HEADROOM = 4096 @@ -86,8 +84,7 @@ class ScoringHeadroomError(ValueError): class ContextNeed: """The context window one task needs from its candidate root model. - *tokens* counts the prompt for ``min_context`` and ``requires``; for - ``max_tokens`` it is the output alone (:func:`fits`). + *tokens* counts the prompt, except for ``max_tokens`` (:func:`fits`). """ root: str @@ -96,14 +93,9 @@ class ContextNeed: def fits(need: ContextNeed, window: int) -> bool: - """Whether *window* serves *need*: a need that counts the prompt - (``min_context``, ``requires``) at most *window*; a ``max_tokens`` below it, - leaving at least one token for the prompt. A longer prompt overflows per - sample (``context_overflow``). - - The one rule for choosing a window (:func:`plan_serving_groups`) and for - checking one (:func:`check_context_fits`). - """ + """Whether *window* serves *need*: ``need <= window`` for a need that counts + the prompt, ``max_tokens < window`` otherwise. The one rule for choosing + and checking a window.""" if need.source == "max_tokens": return need.tokens < window return need.tokens <= window @@ -292,9 +284,8 @@ def parse_model_context(name: str, config: Mapping[str, Any]) -> ModelContext: def task_context_needs(config: Mapping[str, Any]) -> dict[str, ContextNeed]: """The context window each task needs, by task name. - ``min_context`` wins when set. Otherwise the stricter of a declared need - (which counts the prompt) and the effective ``max_tokens`` (which does not; - :func:`fits`). A task with none of the three is left out. + ``min_context`` wins when set, else the stricter of a declared need and the + effective ``max_tokens``. A task with none of the three is left out. """ models = validate_named_config_map("models", config.get("models", {})) tasks = validate_named_config_map("tasks", config.get("tasks", {})) @@ -364,9 +355,8 @@ def check_context_fits( infer_plans: Mapping[str, Mapping[str, Any]] | None, tasks: frozenset[str] | None = None, ) -> list[str]: - """Refuse a task its model's window does not :func:`fit `; return a - warning for each ``max_tokens`` task left less than - :data:`PROMPT_HEADROOM` tokens for its prompt. + """Refuse a task its model's window does not :func:`fit `; return the + prompt-room warnings (:func:`prompt_room_warnings`). A model ``sieval run`` serves is checked against its plan; an endpoint against its root model's ``context_limit``. A model whose window is unknown @@ -385,13 +375,13 @@ def check_context_fits( def prompt_room_warnings( config: Mapping[str, Any], infer_plans: Mapping[str, Mapping[str, Any]] | None, - tasks: frozenset[str] | None = None, + tasks: frozenset[str] | None, *, - launched: bool | None = None, + launched: bool, ) -> list[str]: - """The warnings :func:`check_context_fits` returns, with no refusal: - *launched* ``True`` for the models *infer_plans* serves only, ``False`` for - the endpoints only, ``None`` for both.""" + """A warning per ``max_tokens`` task left under :data:`PROMPT_HEADROOM` + tokens of prompt room, with no refusal: for the models *infer_plans* serves + when *launched*, else for the endpoints.""" return _context_fits(config, infer_plans, tasks, launched)[1] @@ -438,17 +428,19 @@ def _context_fits( def _prompt_room_warning(task: str, need: ContextNeed, window: int) -> str | None: - """A warning when *need* is a ``max_tokens`` that leaves the prompt less - than :data:`PROMPT_HEADROOM` tokens of *window*.""" - room = window - need.tokens - if need.source != "max_tokens" or room >= PROMPT_HEADROOM: + if not _short_of_room(need, window): return None + room = window - need.tokens return ( f"task '{task}' leaves {room} tokens for the prompt at window {window}; " "longer prompts overflow per sample" ) +def _short_of_room(need: ContextNeed, window: int) -> bool: + return need.source == "max_tokens" and window - need.tokens < PROMPT_HEADROOM + + def plan_serving_groups( config: Mapping[str, Any], base_plans: Mapping[str, DeploymentPlan], @@ -456,14 +448,13 @@ def plan_serving_groups( ) -> list[ServingGroup]: """Group the tasks by the plan that serves them, the default group first. - *base_plans* are the recipe plans by root model. Every window chosen is - declared, never sized to a need (:func:`fits` decides): a recipe window - that fits every task stays; else the native window, when it fits the - task; else ``context.max``, in the group ``@`` with - ``context.scaling`` (or unscaled with ``context.extrapolate``). A window - the user pinned is never raised. A plan that serves a task scoring input - then reserves memory for its logits - (:func:`reserve_scoring_headroom`, sized for *gpu_memory_mib* per GPU). + *base_plans* are the recipe plans by root model. A window chosen is a + declared one (:func:`fits` decides): the recipe window; else native, also + when the recipe window leaves a task short of prompt room; else + ``context.max`` in the group ``@``. A pinned window is never + raised. A plan that serves a task scoring input then reserves memory for + its logits (:func:`reserve_scoring_headroom`, sized for *gpu_memory_mib* + per GPU). Raises :class:`ServingShortfallError` naming every task no plan can serve. """ models = validate_named_config_map("models", config.get("models", {})) @@ -508,7 +499,16 @@ def with_headroom( scaled: list[str] = [] for task_name in served[root]: need = needs.get(task_name) - if need is None or window is None or fits(need, window): + if need is None or window is None: + continue + if fits(need, window): + if ( + pinned is None + and native is not None + and window < native + and _short_of_room(need, window) + ): + raise_to_native = True continue if pinned is not None: checked, limit = pinned[1], f"pins {pinned[0]}: {pinned[1]}" @@ -550,7 +550,8 @@ def with_headroom( shortfalls.append(_shortfall(task_name, need, checked, limit)) default_plan = plan - if raise_to_native and native is not None: + if raise_to_native: + assert native is not None default_plan = with_window(plan, native, {}) default_window = served_window(default_plan, native) stays = [task for task in served[root] if task not in scaled] @@ -569,7 +570,8 @@ def with_headroom( ) for task_name in stays: windows[task_name] = stay_served - if scaled and context.max is not None: + if scaled: + assert context.max is not None # scaled only after fits(need, max) key = f"{root}@{context.max}" scaled_plan, headroom = with_headroom( root, with_window(plan, context.max, context.scaling), scaled @@ -723,16 +725,13 @@ def _longest_prompt( tasks: Iterable[str], needs: Mapping[str, ContextNeed], window: int | None ) -> int | None: """The longest prompt *tasks* may score, as far as it is known: each one's - context need at most *window*, or what a ``max_tokens`` leaves of *window*; - ``None`` when a task's is unknown.""" + context need at most *window*; ``None`` when a task's is unknown. A + ``max_tokens`` need bounds nothing: scoring calls send their own.""" bounds: list[int] = [] for task in tasks: need = needs.get(task) - if need is not None and need.source == "max_tokens": - known = [window - need.tokens] if window is not None else [] - else: - tokens = need.tokens if need is not None else None - known = [v for v in (tokens, window) if v is not None] + tokens = need.tokens if need and need.source != "max_tokens" else None + known = [v for v in (tokens, window) if v is not None] if not known: return None bounds.append(min(known)) @@ -780,8 +779,7 @@ def with_window( def _shortfall(task: str, need: ContextNeed, window: int, limit: str) -> str: - """One refused task: what :func:`fits` checked at *window*, the largest - the model could serve it at, and *limit*, why no larger one.""" + """One refused task: what :func:`fits` checked at *window*, and *limit*.""" if need.source == "max_tokens": checked = ( f"max_tokens {need.tokens} leaves no room for the prompt at window {window}" diff --git a/tests/unit/cli/test_serving.py b/tests/unit/cli/test_serving.py index e903a8f3..6801305b 100644 --- a/tests/unit/cli/test_serving.py +++ b/tests/unit/cli/test_serving.py @@ -276,6 +276,22 @@ def test_max_tokens_below_native_is_never_scaled(self) -> None: DEFAULT_GROUP, 40960, "native", 40960, 131072 ) + @pytest.mark.parametrize( + ("max_tokens", "window"), [(28672, 32768), (28673, 40960), (32767, 40960)] + ) + def test_recipe_window_short_of_prompt_room_is_raised_to_native( + self, max_tokens: int, window: int + ) -> None: + """Under 4096 tokens of room at the recipe window, native leaves more.""" + config = _config( + {"t": _gen(max_tokens)}, + model={"path": "/models/m", "context": {"native": 40960}}, + ) + + (group,) = plan_serving_groups(config, {"m": _plan(32768)}) + + assert plan_window(group.plans["m"]) == window + def test_max_tokens_at_or_above_native_scales_to_max(self) -> None: """At native, max_tokens leaves no prompt: the task is scaled, and the scaled window is context.max, not the largest need.""" @@ -766,13 +782,16 @@ def test_warnings_split_by_who_serves_the_model(self) -> None: plans = {"m": _served(40960)} - def tasks(launched: bool | None) -> list[str]: - warnings = prompt_room_warnings(config, plans, launched=launched) + def tasks(launched: bool) -> list[str]: + warnings = prompt_room_warnings(config, plans, None, launched=launched) return [w.split("'")[1] for w in warnings] assert tasks(False) == ["u"] assert tasks(True) == ["t"] - assert tasks(None) == ["t", "u"] + assert [w.split("'")[1] for w in check_context_fits(config, plans)] == [ + "t", + "u", + ] def test_unknown_window_is_not_checked(self) -> None: config = _config({"long": _gen(1_000_000)}, model=_endpoint()) @@ -1073,21 +1092,23 @@ def test_the_longest_scored_prompt_is_the_need_within_the_window( assert group.headroom["m"].prefill_tokens == 2048 - @pytest.mark.parametrize(("max_tokens", "prefill"), [(30720, 2048), (1024, 3072)]) - def test_a_max_tokens_bounds_the_scored_prompt_by_the_room_it_leaves( - self, tmp_path: Path, max_tokens: int, prefill: int + @pytest.mark.parametrize("max_tokens", [None, 1024, 30720, 32000]) + def test_a_max_tokens_does_not_bound_the_scored_prompt( + self, tmp_path: Path, max_tokens: int | None ) -> None: - """32768 - 30720 leaves 2048; 31744 left is above the GPU budget.""" + """Scoring calls send their own max_tokens, so a generation one the + model's args set leaves the prompt the whole window.""" checkpoint = _checkpoint(tmp_path) plan = _plan(checkpoint=checkpoint, mem_fraction_static=0.87) + args = {} if max_tokens is None else {"max_tokens": max_tokens} config = _config( - {"ppl": {"class": SCORING, **_gen(max_tokens)}}, - model={"path": checkpoint}, + {"ppl": {"class": SCORING}, "gen": {}}, + model={"path": checkpoint, "args": args}, ) (group,) = plan_serving_groups(config, {"m": plan}, H100_MIB) - assert group.headroom["m"].prefill_tokens == prefill + assert group.headroom["m"].prefill_tokens == 3072 def test_unknown_gpu_memory_reports_the_unchanged_plan( self, tmp_path: Path