From 53f67b73d1852d28512920df3cf4e452ba64edff Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 14 Nov 2026 16:22:27 +0800 Subject: [PATCH 1/2] feat(core): record exact error statuses and provider error codes Request outcomes now spell every error status as its code ("401", "402", "502", ...) instead of folding them into 4xx/5xx, and each attempt records the provider's own error code from its body (error.status, error.code, error.type, or a vLLM/SGLang top-level type). The request summaries gain error_codes beside outcomes. Readers group codes into actions; SiEval no longer pre-collapses them. Closes #293 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 + docs/guide/configuration.md | 2 +- sieval/core/models/request_log.py | 90 +++- sieval/core/tasks/requests.py | 9 + sieval/core/tasks/run_meta.py | 11 +- tests/integration/test_request_records.py | 8 +- tests/unit/core/models/test_overflow.py | 2 +- tests/unit/core/models/test_request_log.py | 526 ++++++++++++++++++++- tests/unit/core/tasks/test_requests.py | 33 +- 9 files changed, 666 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 049fcf43..51ab3e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Changed + +- **Request outcomes name the exact HTTP status; readers matching `4xx` or `5xx` must adapt (#293).** In `requests.jsonl` and the request summaries (`profile.json` `requests`, `report.json` `sieval.requests`, `meta.json` `runs[].requests`), an error status is now its own code: `400`, `401`, `402`, `403`, `404`, `408`, `413`, `422`, `429`, `500`, `502`, `503`, `504` and any other. The `4xx` and `5xx` keys are gone; group the codes yourself (for example 401/403 as a refused key, 402 as an exhausted balance, 502/503/504 as a gateway or provider error). `2xx`, `context_overflow` (which still wins over the status), `timeout_`, `connection_error`, `cancelled` and `other` are unchanged, and `outcomes` still partitions attempts, so the sum of the non-`2xx` keys is the same failure count as before. Directories written by 0.16 and earlier keep their `4xx`/`5xx` keys and resume as before: `RESULT_SCHEMA_VERSION` stays 1, since no shard, `.idx`, score or identity changes and no resume reads the request counts. + +### Added + +- **The provider's own error code, per attempt and summed (#293).** Each `requests.jsonl` line gains `error_code`, read from the error body when the provider names one: OpenAI and OpenAI-compatible `error.code`, else `error.type` (`insufficient_quota`, `invalid_api_key`, New API's `insufficient_user_quota`); vLLM's and SGLang's top-level `type` (`BadRequestError`) when the only `code` is the status number; Anthropic's `error.type` (`authentication_error`, `overloaded_error`); Google's `error.status` (`PERMISSION_DENIED`, `RESOURCE_EXHAUSTED`). A proxy's text or HTML page, free text, or a code that only repeats the status records `null`. The request summaries gain `error_codes`, counted per phase and model beside `outcomes` (`{"insufficient_quota": 197}`); it covers only attempts that carry a code. The message itself is never recorded. Error bodies are read once by the recording client and stay readable to the SDK and dialects, so no extra request is made. + ## [0.16.0] - 2026-11-10 A minor bump with a breaking change: one rule now decides whether a context diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index f5b789c9..19043ddb 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -349,7 +349,7 @@ legacy shape adapter for an already-bound model and never changes its dialect. Built-in anomaly detection runs automatically after each task and saves per-sample detail to `anomalies.json`; the count is in `report.json`'s `sieval` block. Rules are filtered by task tags — custom rules can be added via `@sieval_detection_rule` (see `sieval/core/tasks/anomaly.py`). -`context_overflow` flags a rollout whose prompt the model under test refused as longer than its context window; the sample is scored as an empty answer instead of failing. See [metrics](metrics.md) for the rule, the `sieval.context_overflow` count and how a task opts back into a failure. In `requests.jsonl` the refused attempt keeps its HTTP status and its outcome is `context_overflow` rather than `4xx`. +`context_overflow` flags a rollout whose prompt the model under test refused as longer than its context window; the sample is scored as an empty answer instead of failing. See [metrics](metrics.md) for the rule, the `sieval.context_overflow` count and how a task opts back into a failure. In `requests.jsonl` the refused attempt keeps its HTTP status and its outcome is `context_overflow` rather than the status (`400`, `413`). ## Sharded Persistence diff --git a/sieval/core/models/request_log.py b/sieval/core/models/request_log.py index 62c45a99..8c1699fc 100644 --- a/sieval/core/models/request_log.py +++ b/sieval/core/models/request_log.py @@ -8,7 +8,8 @@ * :func:`observe_call` -- set by ``Model.arun`` around one model call. With either unset nothing is recorded. A record holds no credential, header or -body, and feeds no identity, digest or ``model_calls[]``. A third, +body, and feeds no identity, digest or ``model_calls[]``. Of an error body it +keeps only the provider's own error code (:func:`provider_error_code`). A third, :func:`request_timeout_scope`, carries a call's ``timeout`` to every try. AI-Generated Code - Claude Opus 5.5 (Anthropic) @@ -17,8 +18,10 @@ import contextvars import email.utils import inspect +import json import math import random +import re import time from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping from contextlib import contextmanager @@ -183,6 +186,7 @@ class _Attempt: status: int | None = None error: str | None = None outcome: str | None = None + error_code: str | None = None request_id: str | None = None retry_after_header_s: float | None = None #: When the request headers went out (httpcore's trace). @@ -222,8 +226,8 @@ def mark_context_overflow(self, exc: BaseException) -> None: """Label the last attempt when *exc* refused the prompt as too long. Its own outcome, so ``outcomes`` still partitions attempts: the refusal - counts once, under ``context_overflow`` rather than ``4xx``. The status - stays the one the server sent. + counts once, under ``context_overflow`` rather than its status. The + status and ``error_code`` stay the ones the server sent. """ if not self.attempts or context_overflow_message(exc) is None: @@ -297,6 +301,7 @@ def lines(self) -> Iterator[dict[str, JSONValue]]: "status": attempt.status, "outcome": attempt.outcome, "error": attempt.error, + "error_code": attempt.error_code, "request_id": attempt.request_id, "retry_after_s": ( round(following.started - ended, 6) @@ -434,21 +439,72 @@ def retry_after_header_s(response: httpx.Response) -> float | None: def status_outcome(status: int) -> str: - """The outcome of a status. A refused over-long prompt is relabelled - ``context_overflow`` when its call ends (``CallScope.mark_context_overflow``). + """The outcome of a status: ``2xx``, an error status as its code (``"401"``, + ``"503"``), else ``other``. Readers group codes; this does not. + + A refused over-long prompt is relabelled ``context_overflow`` when its call + ends (``CallScope.mark_context_overflow``). """ if 200 <= status < 300: return "2xx" - if status == 429: - return "429" - if 400 <= status < 500: - return "4xx" - if status >= 500: - return "5xx" + if status >= 400: + return str(status) return "other" +#: What an error code may look like: one token, no spaces, so no free text. +_ERROR_CODE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}") +#: Placeholders some gateways write where no code exists. +_NO_CODE = frozenset({"none", "null", "undefined"}) + + +def provider_error_code(body: bytes, status: int) -> str | None: + """The provider's own code for an error *body*; ``None`` when it has none. + + The ``error`` object's ``status`` (Google: ``PERMISSION_DENIED``), else its + ``code`` (OpenAI, New API: ``insufficient_quota``), else its ``type`` + (Anthropic: ``authentication_error``; OpenAI without a code). With no + ``error`` object, the body's own ``code`` then ``type`` (vLLM, SGLang: + ``{"object": "error", "type": "BadRequestError", "code": 400}``). A Google + stream error is a one-item list. A value that only repeats *status*, is not + one short token, or is a placeholder is skipped; a body that is not JSON (a + proxy's page) has no code. The message is never read. + """ + + try: + payload = json.loads(body) + except ValueError: + return None + if isinstance(payload, list) and payload: + payload = payload[0] + if not isinstance(payload, dict): + return None + error = payload.get("error") + if isinstance(error, dict): + candidates = (error.get("status"), error.get("code"), error.get("type")) + else: + candidates = (payload.get("code"), payload.get("type")) + for value in candidates: + code = _error_code(value, status) + if code is not None: + return code + return None + + +def _error_code(value: object, status: int) -> str | None: + if isinstance(value, bool) or not isinstance(value, str | int): + return None + code = str(value).strip() + if ( + code == str(status) + or code.lower() in _NO_CODE + or _ERROR_CODE.fullmatch(code) is None + ): + return None + return code + + def error_outcome(exc: BaseException) -> tuple[str, str]: """``(error, outcome)`` for an attempt that raised; class names only.""" @@ -594,6 +650,18 @@ async def _send_once( finally: request.extensions = extensions _finish_response(attempt, response) + if response.is_error: + # The caller reads an error body whole anyway (the SDK, every + # dialect); reading it first leaves it cached for them. + try: + await response.aread() + except BaseException as exc: + _finish_error(attempt, exc) + await response.aclose() + raise + attempt.error_code = provider_error_code( + response.content, response.status_code + ) # An error status ends here: a retry loop closes its body only after # the backoff, which would bill the wait as latency. if stream and attempt.outcome == "2xx": diff --git a/sieval/core/tasks/requests.py b/sieval/core/tasks/requests.py index fa502087..29683242 100644 --- a/sieval/core/tasks/requests.py +++ b/sieval/core/tasks/requests.py @@ -42,6 +42,9 @@ class RequestSummary(TypedDict): retried_calls: int max_attempts: int outcomes: dict[str, int] + #: Attempts by the provider's own error code; attempts without one are + #: not counted, so this does not partition attempts. + error_codes: dict[str, int] retry_wait_s_total: float in_flight: dict[str, LimiterPeak] latency: NotRequired[ProfileTimingStats] @@ -63,6 +66,7 @@ class _Totals: retried_calls: int = 0 max_attempts: int = 0 outcomes: dict[str, int] = field(default_factory=lambda: defaultdict(int)) + error_codes: dict[str, int] = field(default_factory=lambda: defaultdict(int)) retry_wait_s_total: float = 0.0 in_flight: dict[str, LimiterPeak] = field(default_factory=dict) latency: list[float] = field(default_factory=list) @@ -81,6 +85,9 @@ def add(self, line: Mapping[str, JSONValue]) -> None: self.max_attempts = max(self.max_attempts, attempts) outcome = line.get("outcome") self.outcomes[outcome if isinstance(outcome, str) else "other"] += 1 + error_code = line.get("error_code") + if isinstance(error_code, str): + self.error_codes[error_code] += 1 wait = line.get("retry_after_s") if isinstance(wait, float | int): self.retry_wait_s_total += wait @@ -116,6 +123,7 @@ def summary(self) -> RequestSummary: retried_calls=self.retried_calls, max_attempts=self.max_attempts, outcomes=dict(sorted(self.outcomes.items())), + error_codes=dict(sorted(self.error_codes.items())), retry_wait_s_total=round(self.retry_wait_s_total, 6), in_flight=dict(sorted(self.in_flight.items())), ) @@ -266,6 +274,7 @@ def counts(self) -> RunRequests | None: calls=full["calls"], attempts=full["attempts"], outcomes=full["outcomes"], + error_codes=full["error_codes"], in_flight=full["in_flight"], ) return by_phase diff --git a/sieval/core/tasks/run_meta.py b/sieval/core/tasks/run_meta.py index dc4224af..0cf67f16 100644 --- a/sieval/core/tasks/run_meta.py +++ b/sieval/core/tasks/run_meta.py @@ -356,11 +356,20 @@ class LimiterPeak(TypedDict): class RequestCounts(TypedDict): - """One phase x model's model requests in a run: calls, attempts, outcomes.""" + """One phase x model's model requests in a run: calls, attempts, outcomes. + + ``outcomes`` partitions attempts: ``2xx``, an error status as its code + (``"401"``, ``"503"``), ``context_overflow``, ``timeout_``, + ``connection_error``, ``cancelled`` or ``other``. ``error_codes`` counts the + attempts whose error body named a provider code (``insufficient_quota``). + Runs by SiEval 0.16 and earlier have no ``error_codes`` and folded every + other error status into ``4xx`` and ``5xx``. + """ calls: int attempts: int outcomes: dict[str, int] + error_codes: NotRequired[dict[str, int]] in_flight: dict[str, LimiterPeak] diff --git a/tests/integration/test_request_records.py b/tests/integration/test_request_records.py index 66494848..7e0e15b0 100644 --- a/tests/integration/test_request_records.py +++ b/tests/integration/test_request_records.py @@ -58,7 +58,10 @@ def before(request: httpx.Request) -> httpx.Response | None: return None if json.loads(request.content)["messages"][-1]["content"] != question: return None - return httpx.Response(400, json={"error": {"message": "bad request"}}) + return httpx.Response( + 400, + json={"error": {"message": "bad request", "type": "invalid_request_error"}}, + ) return before @@ -218,7 +221,8 @@ async def test_a_resume_appends_and_a_fast_resume_adds_nothing( assert runs[0]["requests"]["infer"]["org/candidate"] == { "calls": 2, "attempts": 2, - "outcomes": {"2xx": 1, "4xx": 1}, + "outcomes": {"2xx": 1, "400": 1}, + "error_codes": {"invalid_request_error": 1}, "in_flight": runs[0]["requests"]["infer"]["org/candidate"]["in_flight"], } assert runs[0]["requests"]["judge"]["gemini-judge"]["calls"] == 1 diff --git a/tests/unit/core/models/test_overflow.py b/tests/unit/core/models/test_overflow.py index 1b41929d..a0bd8ee5 100644 --- a/tests/unit/core/models/test_overflow.py +++ b/tests/unit/core/models/test_overflow.py @@ -469,4 +469,4 @@ def record(self, line: dict[str, JSONValue]) -> None: assert overflow["status"] == 400 assert overflow["outcome"] == CONTEXT_OVERFLOW assert busy["status"] == 503 - assert busy["outcome"] == "5xx" + assert busy["outcome"] == "503" diff --git a/tests/unit/core/models/test_request_log.py b/tests/unit/core/models/test_request_log.py index 4fe68b26..ee491102 100644 --- a/tests/unit/core/models/test_request_log.py +++ b/tests/unit/core/models/test_request_log.py @@ -3,11 +3,13 @@ AI-Generated Code - Claude Opus 5.5 (Anthropic) """ +import contextlib import json import math import time import types -from collections.abc import Callable, Iterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping +from pathlib import Path from typing import Any import anyio @@ -22,6 +24,18 @@ AsyncHTTPJSONConnection, ConnectionRequest, ) +from sieval.core.models.dialects.anthropic_messages import AnthropicMessagesDialect +from sieval.core.models.dialects.google_genai import GoogleGenAIDialect +from sieval.core.models.dialects.sglang_native import SglangNativeDialect +from sieval.core.models.ir import ( + ChatInput, + ChatMessage, + CompletionInput, + Request, + SamplingParams, + SchedulingParams, + TextPart, +) from sieval.core.models.request_log import ( RecordingAsyncClient, RecordingOpenAIClient, @@ -30,6 +44,7 @@ stage_scope, timeout_phase, ) +from sieval.core.tasks.requests import TaskRequestLog from sieval.core.types import JSONValue SECRET = "sk-SENTINEL-NEVER-RECORDED" @@ -163,7 +178,7 @@ async def test_each_sdk_retry_is_its_own_line(self) -> None: assert [line["attempt"] for line in sink.lines] == [1, 2, 3] assert {line["attempts"] for line in sink.lines} == {3} assert [line["status"] for line in sink.lines] == [429, 503, 200] - assert [line["outcome"] for line in sink.lines] == ["429", "5xx", "2xx"] + assert [line["outcome"] for line in sink.lines] == ["429", "503", "2xx"] assert [line["request_id"] for line in sink.lines] == [ "req-429", None, @@ -766,7 +781,26 @@ def response(**headers: str) -> httpx.Response: @pytest.mark.parametrize( ("status", "outcome"), - [(200, "2xx"), (429, "429"), (404, "4xx"), (502, "5xx"), (302, "other")], + [ + (200, "2xx"), + (204, "2xx"), + (400, "400"), + (401, "401"), + (402, "402"), + (403, "403"), + (404, "404"), + (408, "408"), + (413, "413"), + (422, "422"), + (429, "429"), + (500, "500"), + (502, "502"), + (503, "503"), + (504, "504"), + (599, "599"), + (100, "other"), + (302, "other"), + ], ) def test_status_outcome(self, status: int, outcome: str) -> None: assert rl.status_outcome(status) == outcome @@ -838,3 +872,489 @@ def test_a_cyclic_cause_chain_terminates(self) -> None: second.__cause__ = first assert timeout_phase(first) is None + + +class TestProviderErrorCode: + """One body shape per provider; the message is never the code.""" + + @pytest.mark.parametrize( + ("status", "body", "code"), + [ + # OpenAI: `code` first, `type` when the code is null. + ( + 429, + { + "error": { + "message": "You exceeded your current quota", + "type": "insufficient_quota", + "param": None, + "code": "insufficient_quota", + } + }, + "insufficient_quota", + ), + ( + 401, + { + "error": { + "message": "Incorrect API key provided: sk-...", + "type": "invalid_request_error", + "code": "invalid_api_key", + } + }, + "invalid_api_key", + ), + ( + 400, + {"error": {"message": "bad", "type": "invalid_request_error"}}, + "invalid_request_error", + ), + ( + 400, + { + "error": { + "message": "x", + "type": "invalid_request_error", + "code": None, + } + }, + "invalid_request_error", + ), + # New API / one-api gateways. + ( + 403, + { + "error": { + "message": "user quota is not enough", + "type": "new_api_error", + "code": "insufficient_user_quota", + } + }, + "insufficient_user_quota", + ), + # LiteLLM: a numeric-string code that is only the status, then type. + ( + 401, + { + "error": { + "message": "Authentication Error, Invalid proxy server token", + "type": "auth_error", + "param": "None", + "code": "401", + } + }, + "auth_error", + ), + ( + 429, + {"error": {"message": "x", "type": "None", "code": "429"}}, + None, + ), + # vLLM / SGLang: a top-level error whose code is the status number. + ( + 400, + { + "object": "error", + "message": "This model's maximum context length is 4096 tokens", + "type": "BadRequestError", + "param": None, + "code": 400, + }, + "BadRequestError", + ), + ( + 404, + { + "object": "error", + "message": "The model `x` does not exist.", + "type": "NotFoundError", + "code": 404, + }, + "NotFoundError", + ), + # Anthropic. + ( + 401, + { + "type": "error", + "error": { + "type": "authentication_error", + "message": "invalid x-api-key", + }, + }, + "authentication_error", + ), + ( + 529, + { + "type": "error", + "error": {"type": "overloaded_error", "message": "Overloaded"}, + }, + "overloaded_error", + ), + # Google: `status`, with `code` only the HTTP number. + ( + 403, + { + "error": { + "code": 403, + "message": "Permission denied on resource project x.", + "status": "PERMISSION_DENIED", + } + }, + "PERMISSION_DENIED", + ), + # A Google stream error is a one-item list. + ( + 429, + [ + { + "error": { + "code": 429, + "message": "Resource has been exhausted", + "status": "RESOURCE_EXHAUSTED", + } + } + ], + "RESOURCE_EXHAUSTED", + ), + # Google without a status: its code, unless only the HTTP status. + (400, {"error": {"code": 400, "message": "bad"}}, None), + (500, {"error": {"code": 13, "message": "internal"}}, "13"), + # No usable code: free text, an empty or non-token value, no error. + (400, {"error": {"message": "x", "type": "bad request: try again"}}, None), + (400, {"error": {"message": "x", "code": ""}}, None), + (400, {"error": {"message": "x", "code": True}}, None), + (400, {"error": {"message": "x", "code": {"nested": 1}}}, None), + (400, {"error": {"message": "x", "code": "a" * 65}}, None), + (400, {"error": {"message": "x", "code": "null"}}, None), + (502, {"error": "plain"}, None), + (400, {"detail": "Not Found"}, None), + (400, [], None), + (400, ["text"], None), + (400, "just a string", None), + ], + ) + def test_reads_each_providers_shape( + self, status: int, body: JSONValue, code: str | None + ) -> None: + assert rl.provider_error_code(json.dumps(body).encode(), status) == code + + @pytest.mark.parametrize( + "body", + [ + b"502 Bad Gateway", + b"upstream connect error or disconnect/reset before headers", + b"", + ], + ) + def test_a_proxys_page_has_no_code(self, body: bytes) -> None: + assert rl.provider_error_code(body, 502) is None + + +def _recorded( + handler: Callable[[httpx.Request], httpx.Response], base_url: str +) -> AsyncHTTPJSONConnection: + return AsyncHTTPJSONConnection( + RecordingAsyncClient(base_url=base_url, transport=httpx.MockTransport(handler)), + "k", + ) + + +async def _in_call(sink: _Sink, call: Callable[[], Awaitable[object]]) -> None: + with ( + stage_scope(_scope(sink)), + observe_call(model_meta=lambda: {"model": "m"}, model={}, n=1, limiters={}), + ): + await call() + + +class TestErrorCodeOnTheWire: + """Each dialect family's error is recorded with its code, and its caller + still reads the whole body: the recording client reads it once, cached.""" + + @pytest.mark.anyio + @pytest.mark.parametrize("endpoint", ["chat", "completions", "responses"]) + @pytest.mark.parametrize("stream", [False, True]) + async def test_the_openai_sdk_still_reads_its_body( + self, endpoint: str, stream: bool + ) -> None: + body = { + "error": { + "message": "You exceeded your current quota", + "type": "insufficient_quota", + "code": "insufficient_quota", + } + } + handler, _ = _replies(httpx.Response(429, json=body)) + sink = _Sink() + calls = { + "chat": lambda c: c.chat.completions.create( + model="m", messages=[{"role": "user", "content": "q"}], stream=stream + ), + "completions": lambda c: c.completions.create( + model="m", prompt="q", stream=stream + ), + "responses": lambda c: c.responses.create( + model="m", input="q", stream=stream + ), + } + async with _sdk(handler, retries=0) as client: + with pytest.raises(openai.RateLimitError) as raised: + await _in_call(sink, lambda: calls[endpoint](client)) + assert raised.value.code == "insufficient_quota" + assert raised.value.body == body["error"] + [line] = sink.lines + assert (line["outcome"], line["error_code"]) == ("429", "insufficient_quota") + + @pytest.mark.anyio + async def test_a_retried_error_keeps_each_attempts_code(self) -> None: + handler, _ = _replies( + httpx.Response( + 503, + json={ + "object": "error", + "type": "ServiceUnavailableError", + "code": 503, + }, + headers={"retry-after-ms": "1"}, + ), + httpx.Response(502, content=b"502 Bad Gateway"), + httpx.Response(200, json=_chat_body()), + ) + sink = _Sink() + async with _sdk(handler, retries=2) as client: + await _call(client, sink) + assert [(line["outcome"], line["error_code"]) for line in sink.lines] == [ + ("503", "ServiceUnavailableError"), + ("502", None), + ("2xx", None), + ] + + @pytest.mark.anyio + @pytest.mark.parametrize("stream", [False, True]) + async def test_anthropic(self, stream: bool) -> None: + body = ( + b'{"type":"error","error":{"type":"authentication_error",' + b'"message":"invalid x-api-key"}}' + ) + connection = _recorded( + lambda _: httpx.Response( + 401, + stream=_OneChunk(body), + headers={"content-type": "application/json"}, + ), + "https://api.anthropic.com/v1/", + ) + dialect = AnthropicMessagesDialect(connection, "claude-sonnet-4-6") + request = Request( + input=ChatInput((ChatMessage("user", (TextPart("q"),)),)), + sampling=SamplingParams(max_tokens=8), + scheduling=SchedulingParams(stream=stream), + ) + sink = _Sink() + with pytest.raises(httpx.HTTPStatusError) as raised: + await _in_call(sink, lambda: dialect.arun(request)) + await connection.aclose() + assert str(raised.value).endswith("\nAnthropic: invalid x-api-key") + [line] = sink.lines + assert (line["outcome"], line["error_code"]) == ("401", "authentication_error") + assert "invalid x-api-key" not in json.dumps(line) + + @pytest.mark.anyio + @pytest.mark.parametrize("stream", [False, True]) + async def test_google_genai(self, stream: bool) -> None: + body = ( + b'{"error":{"code":403,"message":"Permission denied",' + b'"status":"PERMISSION_DENIED"}}' + ) + connection = _recorded( + lambda _: httpx.Response(403, stream=_OneChunk(body)), + "https://generativelanguage.googleapis.com/", + ) + dialect = GoogleGenAIDialect(connection, "gemini-2.5-flash") + request = Request( + input=ChatInput((ChatMessage("user", (TextPart("q"),)),)), + scheduling=SchedulingParams(stream=stream), + ) + sink = _Sink() + with pytest.raises(httpx.HTTPStatusError) as raised: + await _in_call(sink, lambda: dialect.arun(request)) + await connection.aclose() + assert str(raised.value).endswith("\nGoogle: Permission denied") + [line] = sink.lines + assert (line["outcome"], line["error_code"]) == ("403", "PERMISSION_DENIED") + + @pytest.mark.anyio + async def test_sglang_native(self) -> None: + body = ( + b'{"object":"error","message":"context too long",' + b'"type":"BadRequestError","param":null,"code":400}' + ) + connection = _recorded( + lambda _: httpx.Response(400, stream=_OneChunk(body)), + "http://host:30000/v1", + ) + dialect = SglangNativeDialect(connection, "m") + request = Request(input=CompletionInput("q")) + sink = _Sink() + with pytest.raises(httpx.HTTPStatusError) as raised: + await _in_call(sink, lambda: dialect.arun(request)) + await connection.aclose() + # A top-level error has no `error.message`: the body is the reason. + assert str(raised.value).endswith("\nSGLang: " + body.decode()) + [line] = sink.lines + assert (line["outcome"], line["error_code"]) == ("400", "BadRequestError") + + @pytest.mark.anyio + async def test_a_streamed_error_body_is_read_once_and_still_readable( + self, + ) -> None: + body = b'{"error":{"type":"new_api_error","code":"insufficient_user_quota"}}' + reads: list[int] = [] + + class _Counted(httpx.AsyncByteStream): + async def __aiter__(self): + reads.append(1) + yield body + + client = RecordingAsyncClient( + transport=httpx.MockTransport( + lambda _: httpx.Response(402, stream=_Counted()) + ) + ) + sink = _Sink() + + async def call() -> None: + async with client.stream("POST", "https://gw.example/x") as response: + assert await response.aread() == body + + await _in_call(sink, call) + await client.aclose() + assert reads == [1] + [line] = sink.lines + assert (line["outcome"], line["error_code"]) == ( + "402", + "insufficient_user_quota", + ) + + @pytest.mark.anyio + async def test_an_error_body_that_breaks_is_a_connection_error(self) -> None: + class _Broken(httpx.AsyncByteStream): + async def __aiter__(self): + yield b'{"error":' + raise httpx.ReadError("reset") + + client = RecordingAsyncClient( + transport=httpx.MockTransport( + lambda _: httpx.Response(500, stream=_Broken()) + ) + ) + sink = _Sink() + with pytest.raises(httpx.ReadError): + await _in_call( + sink, + lambda: client.send( + client.build_request("POST", "https://gw.example/x"), stream=True + ), + ) + await client.aclose() + [line] = sink.lines + assert line["status"] == 500 + assert line["outcome"] == "500" + assert line["error"] == "exception::ReadError" + assert line["error_code"] is None + assert client._in_flight == 0 + + @pytest.mark.anyio + async def test_a_success_body_is_not_read_early(self) -> None: + client = RecordingAsyncClient( + transport=httpx.MockTransport( + lambda _: httpx.Response(200, stream=_OneChunk(b"data: [DONE]\n\n")) + ) + ) + sink = _Sink() + + async def call() -> None: + response = await client.send( + client.build_request("POST", "https://gw.example/x"), stream=True + ) + assert not response.is_stream_consumed + await response.aclose() + + await _in_call(sink, call) + await client.aclose() + [line] = sink.lines + assert (line["outcome"], line["error_code"]) == ("2xx", None) + + @pytest.mark.anyio + async def test_context_overflow_still_wins_and_keeps_the_code(self) -> None: + handler, _ = _replies( + httpx.Response( + 400, + json={ + "object": "error", + "message": "This model's maximum context length is 4096 tokens.", + "type": "BadRequestError", + "code": 400, + }, + ) + ) + sink = _Sink() + async with _sdk(handler, retries=0) as client: + with pytest.raises(openai.BadRequestError): + await _call(client, sink) + [line] = sink.lines + assert line["status"] == 400 + assert line["outcome"] == "context_overflow" + assert line["error_code"] == "BadRequestError" + + +class _OneChunk(httpx.AsyncByteStream): + """A body that must be read, as a live one is.""" + + def __init__(self, content: bytes) -> None: + self._content = content + + async def __aiter__(self) -> AsyncIterator[bytes]: + yield self._content + + +class TestOutcomesPartitionAttempts: + @pytest.mark.anyio + async def test_every_attempt_lands_in_exactly_one_outcome( + self, tmp_path: Path + ) -> None: + statuses = [400, 401, 402, 403, 404, 408, 413, 422, 429, 500, 502, 503, 504] + # `None` drops the connection. + script: list[httpx.Response | None] = [httpx.Response(s) for s in statuses] + script += [None, httpx.Response(200, json={})] + queue = list(script) + + def handler(request: httpx.Request) -> httpx.Response: + item = queue.pop(0) + if item is None: + raise httpx.ConnectError("x", request=request) + return item + + client = RecordingAsyncClient(transport=httpx.MockTransport(handler)) + sink = _Sink() + for _ in script: + with contextlib.suppress(httpx.TransportError): + await _in_call( + sink, lambda: client.post("https://gw.example/x", json={}) + ) + await client.aclose() + outcomes = [line["outcome"] for line in sink.lines] + assert outcomes == [str(s) for s in statuses] + ["connection_error", "2xx"] + + log = TaskRequestLog( + tmp_path, subject=None, judges={}, max_bytes=1 << 20, max_files=1 + ) + for line in sink.lines: + log.record(line) + counts = log.counts() + assert counts is not None + [summary] = counts["infer"].values() + assert sum(summary["outcomes"].values()) == summary["attempts"] == len(script) + assert all(n == 1 for n in summary["outcomes"].values()) diff --git a/tests/unit/core/tasks/test_requests.py b/tests/unit/core/tasks/test_requests.py index 26226a56..500c6d2e 100644 --- a/tests/unit/core/tasks/test_requests.py +++ b/tests/unit/core/tasks/test_requests.py @@ -88,9 +88,31 @@ def test_counts_outcomes_latency_waits_and_limiter_peaks( in_flight={"task": {"borrowed": 4, "total": 4}}, ) ) - log.record(_line(attempt=2, attempts=3, outcome="5xx", retry_after_s=2.0)) + log.record( + _line( + attempt=2, + attempts=3, + outcome="503", + error_code="overloaded_error", + retry_after_s=2.0, + ) + ) log.record(_line(attempt=3, attempts=3, latency_s=2.5, queue_s=None)) log.record(_line(phase="judge", model={"id": "j"}, outcome="timeout_read")) + for _ in range(2): + log.record( + _line( + phase="judge", + model={"id": "j"}, + outcome="403", + error_code="insufficient_quota", + ) + ) + # A line without a code, or from before codes were recorded, adds none. + log.record(_line(phase="judge", model={"id": "j"}, outcome="401")) + log.record( + _line(phase="judge", model={"id": "j"}, outcome="401", error_code=None) + ) summary = log.summary() assert summary is not None @@ -101,7 +123,8 @@ def test_counts_outcomes_latency_waits_and_limiter_peaks( assert cand["attempts"] == 3 assert cand["retried_calls"] == 1 assert cand["max_attempts"] == 3 - assert cand["outcomes"] == {"2xx": 1, "429": 1, "5xx": 1} + assert cand["outcomes"] == {"2xx": 1, "429": 1, "503": 1} + assert cand["error_codes"] == {"overloaded_error": 1} assert cand["retry_wait_s_total"] == 3.5 assert cand["in_flight"] == {"task": {"peak": 4, "limit": 4}} assert cand["latency"]["count"] == 3 @@ -110,14 +133,16 @@ def test_counts_outcomes_latency_waits_and_limiter_peaks( assert cand["queue"]["count"] == 2 assert cand["connect"]["count"] == 3 judge = by_phase["judge"]["j"] - assert judge["outcomes"] == {"timeout_read": 1} + assert judge["outcomes"] == {"401": 2, "403": 2, "timeout_read": 1} + assert judge["error_codes"] == {"insufficient_quota": 2} counts = log.counts() assert counts is not None assert counts["infer"]["cand"] == { "calls": 1, "attempts": 3, - "outcomes": {"2xx": 1, "429": 1, "5xx": 1}, + "outcomes": {"2xx": 1, "429": 1, "503": 1}, + "error_codes": {"overloaded_error": 1}, "in_flight": {"task": {"peak": 4, "limit": 4}}, } From c1516cf83cea272d0ae102b0f1bf03683f7d1b0a Mon Sep 17 00:00:00 2001 From: Ethan Date: Sat, 14 Nov 2026 16:51:22 +0800 Subject: [PATCH 2/2] fix(core): skip any HTTP status as a provider error code A gateway wrapping its upstream's status ({"error": {"code": 500}} on a 502) recorded "500" as a provider code. Any number in the HTTP status range is now skipped, so provider_error_code no longer takes the status. Also renames a mislabelled test, notes the early error-body read on retried attempts, and trims the docstrings and CHANGELOG entries. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- sieval/core/models/request_log.py | 44 ++++++++++------------ sieval/core/tasks/requests.py | 3 +- sieval/core/tasks/run_meta.py | 9 ++--- tests/unit/core/models/test_request_log.py | 8 ++-- 5 files changed, 30 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51ab3e50..7d88070d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Changed -- **Request outcomes name the exact HTTP status; readers matching `4xx` or `5xx` must adapt (#293).** In `requests.jsonl` and the request summaries (`profile.json` `requests`, `report.json` `sieval.requests`, `meta.json` `runs[].requests`), an error status is now its own code: `400`, `401`, `402`, `403`, `404`, `408`, `413`, `422`, `429`, `500`, `502`, `503`, `504` and any other. The `4xx` and `5xx` keys are gone; group the codes yourself (for example 401/403 as a refused key, 402 as an exhausted balance, 502/503/504 as a gateway or provider error). `2xx`, `context_overflow` (which still wins over the status), `timeout_`, `connection_error`, `cancelled` and `other` are unchanged, and `outcomes` still partitions attempts, so the sum of the non-`2xx` keys is the same failure count as before. Directories written by 0.16 and earlier keep their `4xx`/`5xx` keys and resume as before: `RESULT_SCHEMA_VERSION` stays 1, since no shard, `.idx`, score or identity changes and no resume reads the request counts. +- **Request outcomes name the exact HTTP status (#293).** In `requests.jsonl` and the request summaries (`profile.json`, `report.json` `sieval.requests`, `meta.json` `runs[].requests`), an error status is its own key (`"401"`, `"402"`, `"503"`, …); `4xx` and `5xx` are gone. Readers matching them must group the codes themselves (`400 <= int(k) < 500`); the non-`2xx` total is unchanged. Older directories keep their keys and resume as before (`RESULT_SCHEMA_VERSION` stays 1: no resume reads these counts). ### Added -- **The provider's own error code, per attempt and summed (#293).** Each `requests.jsonl` line gains `error_code`, read from the error body when the provider names one: OpenAI and OpenAI-compatible `error.code`, else `error.type` (`insufficient_quota`, `invalid_api_key`, New API's `insufficient_user_quota`); vLLM's and SGLang's top-level `type` (`BadRequestError`) when the only `code` is the status number; Anthropic's `error.type` (`authentication_error`, `overloaded_error`); Google's `error.status` (`PERMISSION_DENIED`, `RESOURCE_EXHAUSTED`). A proxy's text or HTML page, free text, or a code that only repeats the status records `null`. The request summaries gain `error_codes`, counted per phase and model beside `outcomes` (`{"insufficient_quota": 197}`); it covers only attempts that carry a code. The message itself is never recorded. Error bodies are read once by the recording client and stay readable to the SDK and dialects, so no extra request is made. +- **The provider's error code, per attempt and summed (#293).** Each `requests.jsonl` line gains `error_code` (`insufficient_quota`, `authentication_error`, `PERMISSION_DENIED`, vLLM's `BadRequestError`), and the request summaries gain `error_codes` beside `outcomes`. Free text, HTTP-status numbers and non-JSON bodies record `null`; the message is never kept. The recording client reads the error body once and the SDK and dialects reuse it. ## [0.16.0] - 2026-11-10 diff --git a/sieval/core/models/request_log.py b/sieval/core/models/request_log.py index 8c1699fc..7665cd7a 100644 --- a/sieval/core/models/request_log.py +++ b/sieval/core/models/request_log.py @@ -439,11 +439,10 @@ def retry_after_header_s(response: httpx.Response) -> float | None: def status_outcome(status: int) -> str: - """The outcome of a status: ``2xx``, an error status as its code (``"401"``, - ``"503"``), else ``other``. Readers group codes; this does not. + """``2xx``, an error status as itself (``"401"``), else ``other``. - A refused over-long prompt is relabelled ``context_overflow`` when its call - ends (``CallScope.mark_context_overflow``). + A refused over-long prompt becomes ``context_overflow`` when its call ends + (``CallScope.mark_context_overflow``). """ if 200 <= status < 300: @@ -453,23 +452,19 @@ def status_outcome(status: int) -> str: return "other" -#: What an error code may look like: one token, no spaces, so no free text. +#: One token, so never free text. _ERROR_CODE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}") -#: Placeholders some gateways write where no code exists. +#: Placeholders some gateways write for "no code". _NO_CODE = frozenset({"none", "null", "undefined"}) -def provider_error_code(body: bytes, status: int) -> str | None: - """The provider's own code for an error *body*; ``None`` when it has none. +def provider_error_code(body: bytes) -> str | None: + """The provider's own code for an error *body*, or ``None``. - The ``error`` object's ``status`` (Google: ``PERMISSION_DENIED``), else its - ``code`` (OpenAI, New API: ``insufficient_quota``), else its ``type`` - (Anthropic: ``authentication_error``; OpenAI without a code). With no - ``error`` object, the body's own ``code`` then ``type`` (vLLM, SGLang: - ``{"object": "error", "type": "BadRequestError", "code": 400}``). A Google - stream error is a one-item list. A value that only repeats *status*, is not - one short token, or is a placeholder is skipped; a body that is not JSON (a - proxy's page) has no code. The message is never read. + Tried in order: ``error.status`` (Google), ``error.code`` (OpenAI), + ``error.type`` (Anthropic); with no ``error`` object, the top-level + ``code`` then ``type`` (vLLM, SGLang). A value counts only as one short + token that is not an HTTP status; the message is never read. """ try: @@ -486,20 +481,21 @@ def provider_error_code(body: bytes, status: int) -> str | None: else: candidates = (payload.get("code"), payload.get("type")) for value in candidates: - code = _error_code(value, status) + code = _error_code(value) if code is not None: return code return None -def _error_code(value: object, status: int) -> str | None: +def _error_code(value: object) -> str | None: if isinstance(value, bool) or not isinstance(value, str | int): return None code = str(value).strip() if ( - code == str(status) + _ERROR_CODE.fullmatch(code) is None or code.lower() in _NO_CODE - or _ERROR_CODE.fullmatch(code) is None + # Any HTTP status, this one or a wrapped upstream's, says nothing new. + or (code.isdigit() and 100 <= int(code) <= 599) ): return None return code @@ -651,17 +647,15 @@ async def _send_once( request.extensions = extensions _finish_response(attempt, response) if response.is_error: - # The caller reads an error body whole anyway (the SDK, every - # dialect); reading it first leaves it cached for them. + # Read once so every attempt, retried or not, records its code; + # the final caller reuses the cached bytes. try: await response.aread() except BaseException as exc: _finish_error(attempt, exc) await response.aclose() raise - attempt.error_code = provider_error_code( - response.content, response.status_code - ) + attempt.error_code = provider_error_code(response.content) # An error status ends here: a retry loop closes its body only after # the backoff, which would bill the wait as latency. if stream and attempt.outcome == "2xx": diff --git a/sieval/core/tasks/requests.py b/sieval/core/tasks/requests.py index 29683242..3aade64a 100644 --- a/sieval/core/tasks/requests.py +++ b/sieval/core/tasks/requests.py @@ -42,8 +42,7 @@ class RequestSummary(TypedDict): retried_calls: int max_attempts: int outcomes: dict[str, int] - #: Attempts by the provider's own error code; attempts without one are - #: not counted, so this does not partition attempts. + #: Attempts by provider error code; not a partition (codeless ones skip). error_codes: dict[str, int] retry_wait_s_total: float in_flight: dict[str, LimiterPeak] diff --git a/sieval/core/tasks/run_meta.py b/sieval/core/tasks/run_meta.py index 0cf67f16..a768175c 100644 --- a/sieval/core/tasks/run_meta.py +++ b/sieval/core/tasks/run_meta.py @@ -358,12 +358,9 @@ class LimiterPeak(TypedDict): class RequestCounts(TypedDict): """One phase x model's model requests in a run: calls, attempts, outcomes. - ``outcomes`` partitions attempts: ``2xx``, an error status as its code - (``"401"``, ``"503"``), ``context_overflow``, ``timeout_``, - ``connection_error``, ``cancelled`` or ``other``. ``error_codes`` counts the - attempts whose error body named a provider code (``insufficient_quota``). - Runs by SiEval 0.16 and earlier have no ``error_codes`` and folded every - other error status into ``4xx`` and ``5xx``. + ``outcomes`` partitions attempts, an error status as itself (``"401"``); + ``error_codes`` counts only attempts with a provider code. Runs by 0.16 and + earlier have no ``error_codes`` and fold statuses into ``4xx``/``5xx``. """ calls: int diff --git a/tests/unit/core/models/test_request_log.py b/tests/unit/core/models/test_request_log.py index ee491102..5621310e 100644 --- a/tests/unit/core/models/test_request_log.py +++ b/tests/unit/core/models/test_request_log.py @@ -1021,6 +1021,8 @@ class TestProviderErrorCode: # Google without a status: its code, unless only the HTTP status. (400, {"error": {"code": 400, "message": "bad"}}, None), (500, {"error": {"code": 13, "message": "internal"}}, "13"), + # A gateway wrapping its upstream's status: still only a status. + (502, {"error": {"code": 500, "message": "upstream"}}, None), # No usable code: free text, an empty or non-token value, no error. (400, {"error": {"message": "x", "type": "bad request: try again"}}, None), (400, {"error": {"message": "x", "code": ""}}, None), @@ -1038,7 +1040,7 @@ class TestProviderErrorCode: def test_reads_each_providers_shape( self, status: int, body: JSONValue, code: str | None ) -> None: - assert rl.provider_error_code(json.dumps(body).encode(), status) == code + assert rl.provider_error_code(json.dumps(body).encode()) == code @pytest.mark.parametrize( "body", @@ -1049,7 +1051,7 @@ def test_reads_each_providers_shape( ], ) def test_a_proxys_page_has_no_code(self, body: bytes) -> None: - assert rl.provider_error_code(body, 502) is None + assert rl.provider_error_code(body) is None def _recorded( @@ -1239,7 +1241,7 @@ async def call() -> None: ) @pytest.mark.anyio - async def test_an_error_body_that_breaks_is_a_connection_error(self) -> None: + async def test_a_broken_error_body_keeps_its_status(self) -> None: class _Broken(httpx.AsyncByteStream): async def __aiter__(self): yield b'{"error":'