feat(usage): show each pooled key's state by slot name

.env.example

@@ -182,6 +182,8 @@ JWT_SECRET=your-jwt-secret-key
 # Provider API Keys (Optional - if you want to use different keys for different providers)
 OPENAI_API_KEY=your-openai-api-key
 ANTHROPIC_API_KEY=your-anthropic-api-key
+# Optional spares, used in order after a key is refused (docs/anthropic.md).
+# ANTHROPIC_API_KEY_1=
 # Required only for Anthropic keys that can act on multiple workspaces.
 ANTHROPIC_WORKSPACE_ID=
 CEREBRAS_API_KEY=your-cerebras-api-key

docs/anthropic.md

@@ -18,6 +18,47 @@ Set `ANTHROPIC_API_KEY` in the server environment or Cloudflare Worker secrets.
 The Worker forwards it to the Container. Restart the server, or deploy the
 updated Worker and Container, after changing configuration.
 
+## Spare keys and failover
+
+Add spare keys as `ANTHROPIC_API_KEY_1`, `ANTHROPIC_API_KEY_2`, and so on:
+
+```bash
+npx wrangler secret put ANTHROPIC_API_KEY_1
+```
+
+Requests use the first key that is not resting, in that order, so the spare
+carries traffic only after the primary is refused. When the Claude API refuses a
+key, the gateway rests it and sends the same request once on the next key. This
+applies to unified routes, native `/anthropic/v1/*` routes, and the
+`auto:intelligence` chain. Refusals that rest a key:
+
+| Response | Rest |
+| --- | --- |
+| 400 `invalid_request_error`, "Your credit balance is too low…" | 1 hour |
+| 400 `invalid_request_error`, "You have reached your specified … API usage limits" | 1 hour |
+| 429 with `error.details.error_code` `enforced_spend_limit_reached` | 1 hour |
+| 402 `billing_error` | 1 hour |
+| 401 `authentication_error` | 5 minutes |
+| 429 rate limit | `retry-after`, or 1 minute, for that model |
+
+These rests apply with `MODEL_COOLDOWN_ENABLED=true`, the production setting,
+and are capped by `MODEL_COOLDOWN_MAX_SECONDS` (default 3600). A 403 can be
+specific to one model, so it and other 400 errors return to the caller without
+trying another key. A rested key returns to service when its rest ends. If every key
+is resting, the request fails with 429 `model_cooldown`. Responses from unified
+routes carry `X-MultiLLM-Credential-Attempts` with the number of keys tried.
+`GET /admin/providers/usage` lists each key's slot name, whether it is the
+active key or resting, and its dispatch and refusal counts; see
+[provider usage](provider-usage.md#credential-slots).
+
+Keys from different Anthropic organizations do not share thinking signatures:
+a signed thinking block replays only in the organization that created it.
+Prompt-cache affinity keeps a conversation on the key that served it, so a
+conversation moves to the spare only when its key is refused. Its next request
+may then fail with an invalid `signature` error; drop the earlier thinking
+blocks, or start a new conversation. `ANTHROPIC_WORKSPACE_ID` is sent with every
+key, so leave it empty when the keys belong to different organizations.
+
 For a key that can act on multiple Anthropic workspaces, also set
 `ANTHROPIC_WORKSPACE_ID` to the intended workspace. The server sends it as
 `anthropic-workspace-id` for generation and model discovery. Caller headers

docs/deployment-cloudflare.md

@@ -255,6 +255,8 @@ Provider secrets are optional and should be set only when used:
 
 ```bash
 npx wrangler secret put OPENAI_API_KEY
+npx wrangler secret put ANTHROPIC_API_KEY
+npx wrangler secret put ANTHROPIC_API_KEY_1
 npx wrangler secret put OPENROUTER_API_KEY
 npx wrangler secret put OPENCODE_GO_API_KEY
 npx wrangler secret put CODEX_EASY_API_KEY

docs/provider-usage.md

@@ -94,6 +94,46 @@ an account-quota API. Its console remains the authoritative source.
 NanoGPT usage lookup follows configured key preference order and advances past
 definitive credential rejections. It does not expose which key succeeded.
 
+## Credential slots
+
+Providers with a key pool (the Claude API and Codex Everywhere) add
+`credential_slots`: one entry per configured key, in selection order, named by
+its environment variable and never by its value.
+
+```json
+"credential_slots": [
+  {
+    "slot": "ANTHROPIC_API_KEY",
+    "active": false,
+    "resting_seconds": 3412,
+    "requests": 41,
+    "errors": 1,
+    "refusals": 1,
+    "last_used_at": "2026-10-26T13:05:12Z",
+    "last_refusal": "anthropic_credit_exhausted",
+    "last_refusal_at": "2026-10-26T13:05:12Z"
+  },
+  {
+    "slot": "ANTHROPIC_API_KEY_1",
+    "active": true,
+    "resting_seconds": 0,
+    "requests": 6,
+    "errors": 0,
+    "refusals": 0,
+    "last_used_at": "2026-10-26T13:09:40Z",
+    "last_refusal": null,
+    "last_refusal_at": null
+  }
+]
+```
+
+`active` marks the first key that is not resting, which new conversations use.
+`resting_seconds` covers rests that apply to every model; a rate-limit rest for a
+single model is not shown. The counters come from the Container process and
+start again at zero when it restarts, for example after a deploy. They count
+dispatches, not spend: the Claude Console's Usage and Cost pages, filtered by
+API key, are the authoritative record of each key's credit.
+
 ## Caching and timeouts
 
 Authoritative results are cached independently per provider. Local request and

providers/anthropic.py

@@ -1,5 +1,6 @@
 """Direct Claude API authentication and model metadata."""
 
+import json
 from collections.abc import Mapping
 from typing import Any
 
@@ -9,6 +10,70 @@
     "anthropic-beta": "Anthropic-Beta",
     "x-client-request-id": "X-Client-Request-ID",
 }
+# An account that ran out of credit or reached a spend limit stays refused until its owner
+# acts, so its key rests for the longest cooldown instead of being retried every minute.
+ANTHROPIC_REFUSAL_REST_SECONDS = 3600
+_REFUSAL_BODY_LIMIT = 65_536
+_SPEND_LIMIT_MESSAGES = (
+    "You have reached your specified API usage limits",
+    "You have reached your specified workspace API usage limits",
+)
+
+
+def _header(headers: Any, name: str) -> str:
+    getter = getattr(headers, "get", None)
+    if getter is None:
+        return ""
+    value = getter(name)
+    if value is None:
+        value = next(
+            (item for key, item in headers.items() if str(key).lower() == name.lower()),
+            None,
+        )
+    return value if isinstance(value, str) else ""
+
+
+def anthropic_credential_refusal(response: Any) -> str | None:
+    """Name the account-wide refusal in a Claude API error response, or return None.
+
+    Anthropic reports an empty credit balance and a spend limit set by the account owner
+    as 400 invalid_request_error, the same shape as a malformed request, so only the
+    message tells them apart. A tier spend cap is a 429 with a distinct error code. Reading
+    the small JSON error body caches it on the response, so relays still forward it.
+    """
+    status = getattr(response, "status_code", None)
+    if status == 402:
+        return "billing_error"
+    if status not in (400, 429):
+        return None
+    headers = getattr(response, "headers", None) or {}
+    if "json" not in _header(headers, "Content-Type").lower():
+        return None
+    length = _header(headers, "Content-Length")
+    if length and (not length.isdigit() or int(length) > _REFUSAL_BODY_LIMIT):
+        return None
+    try:
+        body = response.content
+    except Exception:  # A body that cannot be read is not evidence of a refusal.
+        return None
+    try:
+        error = json.loads(body).get("error") if len(body) <= _REFUSAL_BODY_LIMIT else None
+    except (AttributeError, TypeError, ValueError, RecursionError):
+        return None
+    if not isinstance(error, Mapping):
+        return None
+    if status == 429:
+        details = error.get("details")
+        code = details.get("error_code") if isinstance(details, Mapping) else None
+        return "spend_cap" if code == "enforced_spend_limit_reached" else None
+    message = error.get("message")
+    if error.get("type") != "invalid_request_error" or not isinstance(message, str):
+        return None
+    if message.startswith("Your credit balance is too low"):
+        return "credit_exhausted"
+    if message.startswith(_SPEND_LIMIT_MESSAGES):
+        return "spend_limit"
+    return None
 
 
 def with_anthropic_headers(

routes/provider_credentials.py

@@ -5,6 +5,7 @@
 
 from error_handlers import APIError
 from services.credential_context import observation_context, selection_context
+from services.credential_pool import is_key_rejection
 from services.model_cooldown import ModelCooldownCapacity, ModelCooldownExhausted
 from providers.nanogpt import is_nanogpt_paygo_rejection
 from routes.auto_routes import AutoRouteCandidateUnavailable
@@ -60,6 +61,19 @@ def _is_nanogpt_routing_refusal(response) -> bool:
     )
 
 
+# Providers whose unified requests move to the next configured key after a key refusal.
+ROTATING_PROVIDERS = frozenset({"nanogpt", "anthropic"})
+
+
+def is_anthropic_key_refusal(response) -> bool:
+    """The Claude API refused the key (auth, billing, credit, spend limit or rate limit),
+    so the request generated nothing and may run on the next key. Dispatch already rested
+    the refused key, and the next selection skips it."""
+    return bool(getattr(response, "multillm_credential_refusal", None)) or is_key_rejection(
+        getattr(response, "status_code", None)
+    )
+
+
 def _record_nanogpt_token_result(
     app,
     provider: str,
@@ -96,9 +110,9 @@ def _request_with_provider_token_rotation(
     billing_refusal_is_routing: bool = False,
     *, model=None, quota_bucket=None,
 ):
-    """Send once per usable NanoGPT key after definite pre-generation failures."""
+    """Send once per usable NanoGPT or Claude API key after definite pre-generation failures."""
     attempt_limit = 1
-    if provider == "nanogpt":
+    if provider in ROTATING_PROVIDERS:
         attempt_limit = max(1, len(auth_service_cls.get_api_keys(provider)))
 
     attempted_tokens: set[str] = set()
@@ -147,7 +161,10 @@ def _request_with_provider_token_rotation(
             # Every key would refuse this suffix identically, so rotating on
             # would only surface an unrelated key's error to the caller.
             break
-        if provider != "nanogpt" or not is_nanogpt_credential_rejection(
+        if provider == "anthropic":
+            if not is_anthropic_key_refusal(response):
+                break
+        elif provider != "nanogpt" or not is_nanogpt_credential_rejection(
             response.status_code
         ):
             break
@@ -162,6 +179,6 @@ def _add_credential_attempt_headers(
     provider: str,
     attempts: int,
 ) -> Response:
-    if provider == "nanogpt":
+    if provider in ROTATING_PROVIDERS:
         response.headers["X-MultiLLM-Credential-Attempts"] = str(attempts)
     return response

routes/proxy.py

@@ -51,8 +51,10 @@
     stream_upstream_response,
 )
 from routes.dashboard_chat import register_dashboard_chat_routes
+from routes.provider_credentials import is_anthropic_key_refusal
 from services.nanogpt_key_pool import NanoGPTKeyPool, NanoGPTKeyPoolExhausted
 from services.credential_context import dispatch_context, observation_context, proxy_selection_context
+from services.model_cooldown import ModelCooldownCapacity, ModelCooldownExhausted
 from services.model_cooldown import settings as cooldown_settings
 from services.nanogpt_speed_breaker import NanoGPTSpeedBreaker
 from services.provider_access_policy import provider_route_scope
@@ -445,6 +447,24 @@ def send_to_url(target_url: str):
             else:
                 response = send_to_url(url)
 
+            if api_provider == "anthropic":
+                # A refused Claude API key generated nothing: send once on each other key.
+                # Dispatch rested the refused key, so the next selection skips it.
+                attempted_tokens = {auth_token}
+                while is_anthropic_key_refusal(response):
+                    try:
+                        next_token = auth_service_cls.get_api_key(api_provider, **credential_scope)
+                    except (ModelCooldownExhausted, ModelCooldownCapacity):
+                        break
+                    if not next_token or next_token in attempted_tokens:
+                        break
+                    attempted_tokens.add(next_token)
+                    response.close()
+                    headers = proxy_service_cls.prepare_headers(
+                        request.headers, api_provider, next_token, upstream_path=path,
+                    )
+                    response = send_to_url(url)
+
             if speed_routed_body is not None and is_nanogpt_paygo_rejection(
                 getattr(response, "status_code", None),
                 getattr(response, "content", None),

services/auth_primitives.py

@@ -72,6 +72,8 @@ def provider_credential_env_names(provider: str) -> tuple[str, ...]:
         )
     if provider == "groq":
         return ("GROQ_API_KEY", "GROQ_API_KEY_1", "GROQ_API_KEY_2")
+    if provider == "anthropic":
+        return ("ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY_1")
     if provider == "nanogpt":
         return (
             "NANOGPT_API_KEY",