Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/).

## [Unreleased]

### Changed

- **Request outcomes name the exact HTTP status (#293).** In `requests.jsonl` and the request summaries (`profile.json`, `report.json` `sieval.requests`, `meta.json` `runs[].requests`), an error status is its own key (`"401"`, `"402"`, `"503"`, …); `4xx` and `5xx` are gone. Readers matching them must group the codes themselves (`400 <= int(k) < 500`); the non-`2xx` total is unchanged. Older directories keep their keys and resume as before (`RESULT_SCHEMA_VERSION` stays 1: no resume reads these counts).

### Added

- **The provider's error code, per attempt and summed (#293).** Each `requests.jsonl` line gains `error_code` (`insufficient_quota`, `authentication_error`, `PERMISSION_DENIED`, vLLM's `BadRequestError`), and the request summaries gain `error_codes` beside `outcomes`. Free text, HTTP-status numbers and non-JSON bodies record `null`; the message is never kept. The recording client reads the error body once and the SDK and dialects reuse it.

## [0.16.0] - 2026-11-10

A minor bump with a breaking change: one rule now decides whether a context
Expand Down
2 changes: 1 addition & 1 deletion docs/guide/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,7 @@ legacy shape adapter for an already-bound model and never changes its dialect.

Built-in anomaly detection runs automatically after each task and saves per-sample detail to `anomalies.json`; the count is in `report.json`'s `sieval` block. Rules are filtered by task tags — custom rules can be added via `@sieval_detection_rule` (see `sieval/core/tasks/anomaly.py`).

`context_overflow` flags a rollout whose prompt the model under test refused as longer than its context window; the sample is scored as an empty answer instead of failing. See [metrics](metrics.md) for the rule, the `sieval.context_overflow` count and how a task opts back into a failure. In `requests.jsonl` the refused attempt keeps its HTTP status and its outcome is `context_overflow` rather than `4xx`.
`context_overflow` flags a rollout whose prompt the model under test refused as longer than its context window; the sample is scored as an empty answer instead of failing. See [metrics](metrics.md) for the rule, the `sieval.context_overflow` count and how a task opts back into a failure. In `requests.jsonl` the refused attempt keeps its HTTP status and its outcome is `context_overflow` rather than the status (`400`, `413`).

## Sharded Persistence

Expand Down
84 changes: 73 additions & 11 deletions sieval/core/models/request_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
* :func:`observe_call` -- set by ``Model.arun`` around one model call.

With either unset nothing is recorded. A record holds no credential, header or
body, and feeds no identity, digest or ``model_calls[]``. A third,
body, and feeds no identity, digest or ``model_calls[]``. Of an error body it
keeps only the provider's own error code (:func:`provider_error_code`). A third,
:func:`request_timeout_scope`, carries a call's ``timeout`` to every try.

AI-Generated Code - Claude Opus 5.5 (Anthropic)
Expand All @@ -17,8 +18,10 @@
import contextvars
import email.utils
import inspect
import json
import math
import random
import re
import time
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator, Mapping
from contextlib import contextmanager
Expand Down Expand Up @@ -183,6 +186,7 @@ class _Attempt:
status: int | None = None
error: str | None = None
outcome: str | None = None
error_code: str | None = None
request_id: str | None = None
retry_after_header_s: float | None = None
#: When the request headers went out (httpcore's trace).
Expand Down Expand Up @@ -222,8 +226,8 @@ def mark_context_overflow(self, exc: BaseException) -> None:
"""Label the last attempt when *exc* refused the prompt as too long.

Its own outcome, so ``outcomes`` still partitions attempts: the refusal
counts once, under ``context_overflow`` rather than ``4xx``. The status
stays the one the server sent.
counts once, under ``context_overflow`` rather than its status. The
status and ``error_code`` stay the ones the server sent.
"""

if not self.attempts or context_overflow_message(exc) is None:
Expand Down Expand Up @@ -297,6 +301,7 @@ def lines(self) -> Iterator[dict[str, JSONValue]]:
"status": attempt.status,
"outcome": attempt.outcome,
"error": attempt.error,
"error_code": attempt.error_code,
"request_id": attempt.request_id,
"retry_after_s": (
round(following.started - ended, 6)
Expand Down Expand Up @@ -434,21 +439,68 @@ def retry_after_header_s(response: httpx.Response) -> float | None:


def status_outcome(status: int) -> str:
"""The outcome of a status. A refused over-long prompt is relabelled
``context_overflow`` when its call ends (``CallScope.mark_context_overflow``).
"""``2xx``, an error status as itself (``"401"``), else ``other``.

A refused over-long prompt becomes ``context_overflow`` when its call ends
(``CallScope.mark_context_overflow``).
"""

if 200 <= status < 300:
return "2xx"
if status == 429:
return "429"
if 400 <= status < 500:
return "4xx"
if status >= 500:
return "5xx"
if status >= 400:
return str(status)
return "other"


#: One token, so never free text.
_ERROR_CODE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}")
#: Placeholders some gateways write for "no code".
_NO_CODE = frozenset({"none", "null", "undefined"})


def provider_error_code(body: bytes) -> str | None:
"""The provider's own code for an error *body*, or ``None``.

Tried in order: ``error.status`` (Google), ``error.code`` (OpenAI),
``error.type`` (Anthropic); with no ``error`` object, the top-level
``code`` then ``type`` (vLLM, SGLang). A value counts only as one short
token that is not an HTTP status; the message is never read.
"""

try:
payload = json.loads(body)
except ValueError:
return None
if isinstance(payload, list) and payload:
payload = payload[0]
if not isinstance(payload, dict):
return None
error = payload.get("error")
if isinstance(error, dict):
candidates = (error.get("status"), error.get("code"), error.get("type"))
else:
candidates = (payload.get("code"), payload.get("type"))
for value in candidates:
code = _error_code(value)
if code is not None:
return code
return None


def _error_code(value: object) -> str | None:
if isinstance(value, bool) or not isinstance(value, str | int):
return None
code = str(value).strip()
if (
_ERROR_CODE.fullmatch(code) is None
or code.lower() in _NO_CODE
# Any HTTP status, this one or a wrapped upstream's, says nothing new.
or (code.isdigit() and 100 <= int(code) <= 599)
):
return None
return code


def error_outcome(exc: BaseException) -> tuple[str, str]:
"""``(error, outcome)`` for an attempt that raised; class names only."""

Expand Down Expand Up @@ -594,6 +646,16 @@ async def _send_once(
finally:
request.extensions = extensions
_finish_response(attempt, response)
if response.is_error:
# Read once so every attempt, retried or not, records its code;
# the final caller reuses the cached bytes.
try:
await response.aread()
except BaseException as exc:
_finish_error(attempt, exc)
await response.aclose()
raise
attempt.error_code = provider_error_code(response.content)
# An error status ends here: a retry loop closes its body only after
# the backoff, which would bill the wait as latency.
if stream and attempt.outcome == "2xx":
Expand Down
8 changes: 8 additions & 0 deletions sieval/core/tasks/requests.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ class RequestSummary(TypedDict):
retried_calls: int
max_attempts: int
outcomes: dict[str, int]
#: Attempts by provider error code; not a partition (codeless ones skip).
error_codes: dict[str, int]
retry_wait_s_total: float
in_flight: dict[str, LimiterPeak]
latency: NotRequired[ProfileTimingStats]
Expand All @@ -63,6 +65,7 @@ class _Totals:
retried_calls: int = 0
max_attempts: int = 0
outcomes: dict[str, int] = field(default_factory=lambda: defaultdict(int))
error_codes: dict[str, int] = field(default_factory=lambda: defaultdict(int))
retry_wait_s_total: float = 0.0
in_flight: dict[str, LimiterPeak] = field(default_factory=dict)
latency: list[float] = field(default_factory=list)
Expand All @@ -81,6 +84,9 @@ def add(self, line: Mapping[str, JSONValue]) -> None:
self.max_attempts = max(self.max_attempts, attempts)
outcome = line.get("outcome")
self.outcomes[outcome if isinstance(outcome, str) else "other"] += 1
error_code = line.get("error_code")
if isinstance(error_code, str):
self.error_codes[error_code] += 1
wait = line.get("retry_after_s")
if isinstance(wait, float | int):
self.retry_wait_s_total += wait
Expand Down Expand Up @@ -116,6 +122,7 @@ def summary(self) -> RequestSummary:
retried_calls=self.retried_calls,
max_attempts=self.max_attempts,
outcomes=dict(sorted(self.outcomes.items())),
error_codes=dict(sorted(self.error_codes.items())),
retry_wait_s_total=round(self.retry_wait_s_total, 6),
in_flight=dict(sorted(self.in_flight.items())),
)
Expand Down Expand Up @@ -266,6 +273,7 @@ def counts(self) -> RunRequests | None:
calls=full["calls"],
attempts=full["attempts"],
outcomes=full["outcomes"],
error_codes=full["error_codes"],
in_flight=full["in_flight"],
)
return by_phase
8 changes: 7 additions & 1 deletion sieval/core/tasks/run_meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,11 +356,17 @@ class LimiterPeak(TypedDict):


class RequestCounts(TypedDict):
"""One phase x model's model requests in a run: calls, attempts, outcomes."""
"""One phase x model's model requests in a run: calls, attempts, outcomes.

``outcomes`` partitions attempts, an error status as itself (``"401"``);
``error_codes`` counts only attempts with a provider code. Runs by 0.16 and
earlier have no ``error_codes`` and fold statuses into ``4xx``/``5xx``.
"""

calls: int
attempts: int
outcomes: dict[str, int]
error_codes: NotRequired[dict[str, int]]
in_flight: dict[str, LimiterPeak]


Expand Down
8 changes: 6 additions & 2 deletions tests/integration/test_request_records.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,10 @@ def before(request: httpx.Request) -> httpx.Response | None:
return None
if json.loads(request.content)["messages"][-1]["content"] != question:
return None
return httpx.Response(400, json={"error": {"message": "bad request"}})
return httpx.Response(
400,
json={"error": {"message": "bad request", "type": "invalid_request_error"}},
)

return before

Expand Down Expand Up @@ -218,7 +221,8 @@ async def test_a_resume_appends_and_a_fast_resume_adds_nothing(
assert runs[0]["requests"]["infer"]["org/candidate"] == {
"calls": 2,
"attempts": 2,
"outcomes": {"2xx": 1, "4xx": 1},
"outcomes": {"2xx": 1, "400": 1},
"error_codes": {"invalid_request_error": 1},
"in_flight": runs[0]["requests"]["infer"]["org/candidate"]["in_flight"],
}
assert runs[0]["requests"]["judge"]["gemini-judge"]["calls"] == 1
Expand Down
2 changes: 1 addition & 1 deletion tests/unit/core/models/test_overflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -469,4 +469,4 @@ def record(self, line: dict[str, JSONValue]) -> None:
assert overflow["status"] == 400
assert overflow["outcome"] == CONTEXT_OVERFLOW
assert busy["status"] == 503
assert busy["outcome"] == "5xx"
assert busy["outcome"] == "503"
Loading
Loading