diff --git a/CHANGELOG.md b/CHANGELOG.md index 049fcf43..7d88070d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- **Request outcomes name the exact HTTP status (#293).** In `requests.jsonl` and the request summaries (`profile.json`, `report.json` `sieval.requests`, `meta.json` `runs[].requests`), an error status is its own key (`"401"`, `"402"`, `"503"`, …); `4xx` and `5xx` are gone. Readers matching them must group the codes themselves (`400 <= int(k) < 500`); the non-`2xx` total is unchanged. Older directories keep their keys and resume as before (`RESULT_SCHEMA_VERSION` stays 1: no resume reads these counts). + +### Added + +- **The provider's error code, per attempt and summed (#293).** Each `requests.jsonl` line gains `error_code` (`insufficient_quota`, `authentication_error`, `PERMISSION_DENIED`, vLLM's `BadRequestError`), and the request summaries gain `error_codes` beside `outcomes`. Free text, HTTP-status numbers and non-JSON bodies record `null`; the message is never kept. The recording client reads the error body once and the SDK and dialects reuse it. + ## [0.16.0] - 2026-11-10 A minor bump with a breaking change: one rule now decides whether a context diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index f5b789c9..19043ddb 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -349,7 +349,7 @@ legacy shape adapter for an already-bound model and never changes its dialect. Built-in anomaly detection runs automatically after each task and saves per-sample detail to `anomalies.json`; the count is in `report.json`'s `sieval` block. Rules are filtered by task tags — custom rules can be added via `@sieval_detection_rule` (see `sieval/core/tasks/anomaly.py`). -`context_overflow` flags a rollout whose prompt the model under test refused as longer than its context window; the sample is scored as an empty answer instead of failing. See [metrics](metrics.md) for the rule, the `sieval.context_overflow` count and how a task opts back into a failure. In `requests.jsonl` the refused attempt keeps its HTTP status and its outcome is `context_overflow` rather than `4xx`. +`context_overflow` flags a rollout whose prompt the model under test refused as longer than its context window; the sample is scored as an empty answer instead of failing. See [metrics](metrics.md) for the rule, the `sieval.context_overflow` count and how a task opts back into a failure. In `requests.jsonl` the refused attempt keeps its HTTP status and its outcome is `context_overflow` rather than the status (`400`, `413`). ## Sharded Persistence diff --git a/sieval/core/models/request_log.py b/sieval/core/models/request_log.py index 62c45a99..7665cd7a 100644 --- a/sieval/core/models/request_log.py +++ b/sieval/core/models/request_log.py @@ -8,7 +8,8 @@ * :func:`observe_call` -- set by ``Model.arun`` around one model call. With either unset nothing is recorded. A record holds no credential, header or -body, and feeds no identity, digest or ``model_calls[]``. A third, +body, and feeds no identity, digest or ``model_calls[]``. Of an error body it +keeps only the provider's own error code (:func:`provider_error_code`). A third, :func:`request_timeout_scope`, carries a call's ``timeout`` to every try. AI-Generated Code - Claude Opus 5.5 (Anthropic) @@ -17,8 +18,10 @@ import contextvars import email.utils import inspect +import json import math import random +import re import time from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping from contextlib import contextmanager @@ -183,6 +186,7 @@ class _Attempt: status: int | None = None error: str | None = None outcome: str | None = None + error_code: str | None = None request_id: str | None = None retry_after_header_s: float | None = None #: When the request headers went out (httpcore's trace). @@ -222,8 +226,8 @@ def mark_context_overflow(self, exc: BaseException) -> None: """Label the last attempt when *exc* refused the prompt as too long. Its own outcome, so ``outcomes`` still partitions attempts: the refusal - counts once, under ``context_overflow`` rather than ``4xx``. The status - stays the one the server sent. + counts once, under ``context_overflow`` rather than its status. The + status and ``error_code`` stay the ones the server sent. """ if not self.attempts or context_overflow_message(exc) is None: @@ -297,6 +301,7 @@ def lines(self) -> Iterator[dict[str, JSONValue]]: "status": attempt.status, "outcome": attempt.outcome, "error": attempt.error, + "error_code": attempt.error_code, "request_id": attempt.request_id, "retry_after_s": ( round(following.started - ended, 6) @@ -434,21 +439,68 @@ def retry_after_header_s(response: httpx.Response) -> float | None: def status_outcome(status: int) -> str: - """The outcome of a status. A refused over-long prompt is relabelled - ``context_overflow`` when its call ends (``CallScope.mark_context_overflow``). + """``2xx``, an error status as itself (``"401"``), else ``other``. + + A refused over-long prompt becomes ``context_overflow`` when its call ends + (``CallScope.mark_context_overflow``). """ if 200 <= status < 300: return "2xx" - if status == 429: - return "429" - if 400 <= status < 500: - return "4xx" - if status >= 500: - return "5xx" + if status >= 400: + return str(status) return "other" +#: One token, so never free text. +_ERROR_CODE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}") +#: Placeholders some gateways write for "no code". +_NO_CODE = frozenset({"none", "null", "undefined"}) + + +def provider_error_code(body: bytes) -> str | None: + """The provider's own code for an error *body*, or ``None``. + + Tried in order: ``error.status`` (Google), ``error.code`` (OpenAI), + ``error.type`` (Anthropic); with no ``error`` object, the top-level + ``code`` then ``type`` (vLLM, SGLang). A value counts only as one short + token that is not an HTTP status; the message is never read. + """ + + try: + payload = json.loads(body) + except ValueError: + return None + if isinstance(payload, list) and payload: + payload = payload[0] + if not isinstance(payload, dict): + return None + error = payload.get("error") + if isinstance(error, dict): + candidates = (error.get("status"), error.get("code"), error.get("type")) + else: + candidates = (payload.get("code"), payload.get("type")) + for value in candidates: + code = _error_code(value) + if code is not None: + return code + return None + + +def _error_code(value: object) -> str | None: + if isinstance(value, bool) or not isinstance(value, str | int): + return None + code = str(value).strip() + if ( + _ERROR_CODE.fullmatch(code) is None + or code.lower() in _NO_CODE + # Any HTTP status, this one or a wrapped upstream's, says nothing new. + or (code.isdigit() and 100 <= int(code) <= 599) + ): + return None + return code + + def error_outcome(exc: BaseException) -> tuple[str, str]: """``(error, outcome)`` for an attempt that raised; class names only.""" @@ -594,6 +646,16 @@ async def _send_once( finally: request.extensions = extensions _finish_response(attempt, response) + if response.is_error: + # Read once so every attempt, retried or not, records its code; + # the final caller reuses the cached bytes. + try: + await response.aread() + except BaseException as exc: + _finish_error(attempt, exc) + await response.aclose() + raise + attempt.error_code = provider_error_code(response.content) # An error status ends here: a retry loop closes its body only after # the backoff, which would bill the wait as latency. if stream and attempt.outcome == "2xx": diff --git a/sieval/core/tasks/requests.py b/sieval/core/tasks/requests.py index fa502087..3aade64a 100644 --- a/sieval/core/tasks/requests.py +++ b/sieval/core/tasks/requests.py @@ -42,6 +42,8 @@ class RequestSummary(TypedDict): retried_calls: int max_attempts: int outcomes: dict[str, int] + #: Attempts by provider error code; not a partition (codeless ones skip). + error_codes: dict[str, int] retry_wait_s_total: float in_flight: dict[str, LimiterPeak] latency: NotRequired[ProfileTimingStats] @@ -63,6 +65,7 @@ class _Totals: retried_calls: int = 0 max_attempts: int = 0 outcomes: dict[str, int] = field(default_factory=lambda: defaultdict(int)) + error_codes: dict[str, int] = field(default_factory=lambda: defaultdict(int)) retry_wait_s_total: float = 0.0 in_flight: dict[str, LimiterPeak] = field(default_factory=dict) latency: list[float] = field(default_factory=list) @@ -81,6 +84,9 @@ def add(self, line: Mapping[str, JSONValue]) -> None: self.max_attempts = max(self.max_attempts, attempts) outcome = line.get("outcome") self.outcomes[outcome if isinstance(outcome, str) else "other"] += 1 + error_code = line.get("error_code") + if isinstance(error_code, str): + self.error_codes[error_code] += 1 wait = line.get("retry_after_s") if isinstance(wait, float | int): self.retry_wait_s_total += wait @@ -116,6 +122,7 @@ def summary(self) -> RequestSummary: retried_calls=self.retried_calls, max_attempts=self.max_attempts, outcomes=dict(sorted(self.outcomes.items())), + error_codes=dict(sorted(self.error_codes.items())), retry_wait_s_total=round(self.retry_wait_s_total, 6), in_flight=dict(sorted(self.in_flight.items())), ) @@ -266,6 +273,7 @@ def counts(self) -> RunRequests | None: calls=full["calls"], attempts=full["attempts"], outcomes=full["outcomes"], + error_codes=full["error_codes"], in_flight=full["in_flight"], ) return by_phase diff --git a/sieval/core/tasks/run_meta.py b/sieval/core/tasks/run_meta.py index dc4224af..a768175c 100644 --- a/sieval/core/tasks/run_meta.py +++ b/sieval/core/tasks/run_meta.py @@ -356,11 +356,17 @@ class LimiterPeak(TypedDict): class RequestCounts(TypedDict): - """One phase x model's model requests in a run: calls, attempts, outcomes.""" + """One phase x model's model requests in a run: calls, attempts, outcomes. + + ``outcomes`` partitions attempts, an error status as itself (``"401"``); + ``error_codes`` counts only attempts with a provider code. Runs by 0.16 and + earlier have no ``error_codes`` and fold statuses into ``4xx``/``5xx``. + """ calls: int attempts: int outcomes: dict[str, int] + error_codes: NotRequired[dict[str, int]] in_flight: dict[str, LimiterPeak] diff --git a/tests/integration/test_request_records.py b/tests/integration/test_request_records.py index 66494848..7e0e15b0 100644 --- a/tests/integration/test_request_records.py +++ b/tests/integration/test_request_records.py @@ -58,7 +58,10 @@ def before(request: httpx.Request) -> httpx.Response | None: return None if json.loads(request.content)["messages"][-1]["content"] != question: return None - return httpx.Response(400, json={"error": {"message": "bad request"}}) + return httpx.Response( + 400, + json={"error": {"message": "bad request", "type": "invalid_request_error"}}, + ) return before @@ -218,7 +221,8 @@ async def test_a_resume_appends_and_a_fast_resume_adds_nothing( assert runs[0]["requests"]["infer"]["org/candidate"] == { "calls": 2, "attempts": 2, - "outcomes": {"2xx": 1, "4xx": 1}, + "outcomes": {"2xx": 1, "400": 1}, + "error_codes": {"invalid_request_error": 1}, "in_flight": runs[0]["requests"]["infer"]["org/candidate"]["in_flight"], } assert runs[0]["requests"]["judge"]["gemini-judge"]["calls"] == 1 diff --git a/tests/unit/core/models/test_overflow.py b/tests/unit/core/models/test_overflow.py index 1b41929d..a0bd8ee5 100644 --- a/tests/unit/core/models/test_overflow.py +++ b/tests/unit/core/models/test_overflow.py @@ -469,4 +469,4 @@ def record(self, line: dict[str, JSONValue]) -> None: assert overflow["status"] == 400 assert overflow["outcome"] == CONTEXT_OVERFLOW assert busy["status"] == 503 - assert busy["outcome"] == "5xx" + assert busy["outcome"] == "503" diff --git a/tests/unit/core/models/test_request_log.py b/tests/unit/core/models/test_request_log.py index 4fe68b26..5621310e 100644 --- a/tests/unit/core/models/test_request_log.py +++ b/tests/unit/core/models/test_request_log.py @@ -3,11 +3,13 @@ AI-Generated Code - Claude Opus 5.5 (Anthropic) """ +import contextlib import json import math import time import types -from collections.abc import Callable, Iterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping +from pathlib import Path from typing import Any import anyio @@ -22,6 +24,18 @@ AsyncHTTPJSONConnection, ConnectionRequest, ) +from sieval.core.models.dialects.anthropic_messages import AnthropicMessagesDialect +from sieval.core.models.dialects.google_genai import GoogleGenAIDialect +from sieval.core.models.dialects.sglang_native import SglangNativeDialect +from sieval.core.models.ir import ( + ChatInput, + ChatMessage, + CompletionInput, + Request, + SamplingParams, + SchedulingParams, + TextPart, +) from sieval.core.models.request_log import ( RecordingAsyncClient, RecordingOpenAIClient, @@ -30,6 +44,7 @@ stage_scope, timeout_phase, ) +from sieval.core.tasks.requests import TaskRequestLog from sieval.core.types import JSONValue SECRET = "sk-SENTINEL-NEVER-RECORDED" @@ -163,7 +178,7 @@ async def test_each_sdk_retry_is_its_own_line(self) -> None: assert [line["attempt"] for line in sink.lines] == [1, 2, 3] assert {line["attempts"] for line in sink.lines} == {3} assert [line["status"] for line in sink.lines] == [429, 503, 200] - assert [line["outcome"] for line in sink.lines] == ["429", "5xx", "2xx"] + assert [line["outcome"] for line in sink.lines] == ["429", "503", "2xx"] assert [line["request_id"] for line in sink.lines] == [ "req-429", None, @@ -766,7 +781,26 @@ def response(**headers: str) -> httpx.Response: @pytest.mark.parametrize( ("status", "outcome"), - [(200, "2xx"), (429, "429"), (404, "4xx"), (502, "5xx"), (302, "other")], + [ + (200, "2xx"), + (204, "2xx"), + (400, "400"), + (401, "401"), + (402, "402"), + (403, "403"), + (404, "404"), + (408, "408"), + (413, "413"), + (422, "422"), + (429, "429"), + (500, "500"), + (502, "502"), + (503, "503"), + (504, "504"), + (599, "599"), + (100, "other"), + (302, "other"), + ], ) def test_status_outcome(self, status: int, outcome: str) -> None: assert rl.status_outcome(status) == outcome @@ -838,3 +872,491 @@ def test_a_cyclic_cause_chain_terminates(self) -> None: second.__cause__ = first assert timeout_phase(first) is None + + +class TestProviderErrorCode: + """One body shape per provider; the message is never the code.""" + + @pytest.mark.parametrize( + ("status", "body", "code"), + [ + # OpenAI: `code` first, `type` when the code is null. + ( + 429, + { + "error": { + "message": "You exceeded your current quota", + "type": "insufficient_quota", + "param": None, + "code": "insufficient_quota", + } + }, + "insufficient_quota", + ), + ( + 401, + { + "error": { + "message": "Incorrect API key provided: sk-...", + "type": "invalid_request_error", + "code": "invalid_api_key", + } + }, + "invalid_api_key", + ), + ( + 400, + {"error": {"message": "bad", "type": "invalid_request_error"}}, + "invalid_request_error", + ), + ( + 400, + { + "error": { + "message": "x", + "type": "invalid_request_error", + "code": None, + } + }, + "invalid_request_error", + ), + # New API / one-api gateways. + ( + 403, + { + "error": { + "message": "user quota is not enough", + "type": "new_api_error", + "code": "insufficient_user_quota", + } + }, + "insufficient_user_quota", + ), + # LiteLLM: a numeric-string code that is only the status, then type. + ( + 401, + { + "error": { + "message": "Authentication Error, Invalid proxy server token", + "type": "auth_error", + "param": "None", + "code": "401", + } + }, + "auth_error", + ), + ( + 429, + {"error": {"message": "x", "type": "None", "code": "429"}}, + None, + ), + # vLLM / SGLang: a top-level error whose code is the status number. + ( + 400, + { + "object": "error", + "message": "This model's maximum context length is 4096 tokens", + "type": "BadRequestError", + "param": None, + "code": 400, + }, + "BadRequestError", + ), + ( + 404, + { + "object": "error", + "message": "The model `x` does not exist.", + "type": "NotFoundError", + "code": 404, + }, + "NotFoundError", + ), + # Anthropic. + ( + 401, + { + "type": "error", + "error": { + "type": "authentication_error", + "message": "invalid x-api-key", + }, + }, + "authentication_error", + ), + ( + 529, + { + "type": "error", + "error": {"type": "overloaded_error", "message": "Overloaded"}, + }, + "overloaded_error", + ), + # Google: `status`, with `code` only the HTTP number. + ( + 403, + { + "error": { + "code": 403, + "message": "Permission denied on resource project x.", + "status": "PERMISSION_DENIED", + } + }, + "PERMISSION_DENIED", + ), + # A Google stream error is a one-item list. + ( + 429, + [ + { + "error": { + "code": 429, + "message": "Resource has been exhausted", + "status": "RESOURCE_EXHAUSTED", + } + } + ], + "RESOURCE_EXHAUSTED", + ), + # Google without a status: its code, unless only the HTTP status. + (400, {"error": {"code": 400, "message": "bad"}}, None), + (500, {"error": {"code": 13, "message": "internal"}}, "13"), + # A gateway wrapping its upstream's status: still only a status. + (502, {"error": {"code": 500, "message": "upstream"}}, None), + # No usable code: free text, an empty or non-token value, no error. + (400, {"error": {"message": "x", "type": "bad request: try again"}}, None), + (400, {"error": {"message": "x", "code": ""}}, None), + (400, {"error": {"message": "x", "code": True}}, None), + (400, {"error": {"message": "x", "code": {"nested": 1}}}, None), + (400, {"error": {"message": "x", "code": "a" * 65}}, None), + (400, {"error": {"message": "x", "code": "null"}}, None), + (502, {"error": "plain"}, None), + (400, {"detail": "Not Found"}, None), + (400, [], None), + (400, ["text"], None), + (400, "just a string", None), + ], + ) + def test_reads_each_providers_shape( + self, status: int, body: JSONValue, code: str | None + ) -> None: + assert rl.provider_error_code(json.dumps(body).encode()) == code + + @pytest.mark.parametrize( + "body", + [ + b"