feat(lean): keep Haiku coding sessions cached and under the 5x long-prompt price

.env.example

@@ -85,6 +85,19 @@ NANOGPT_KEY_CHECK_EVERY_REQUESTS=50
 NANOGPT_KEY_REJECTED_COOLDOWN_SECONDS=60
 NAVYAI_MAX_REQUEST_BYTES=33554432
 
+# Lean endpoint (/lean/v1/...) for Claude API models; see docs/lean.md.
+# LEAN_ENDPOINT_ENABLED=true
+# LEAN_PROMPT_TOKEN_LIMIT=99000
+# LEAN_PRICE_LIMIT_BASIS=original
+# LEAN_CLEAR_TRIGGER_TOKENS=80000
+# LEAN_CLEAR_KEEP_TOOL_USES=3
+# LEAN_CLEAR_AT_LEAST_TOKENS=20000
+# LEAN_CLEAR_EXCLUDE_TOOLS=
+# LEAN_CACHE_TTL=5m
+# LEAN_CONDENSE_SHELL_OUTPUT=true
+# LEAN_TOOL_RESULT_MAX_CHARS=100000
+# LEAN_STYLE=none
+
 # Process-local recovery for normalized transports. Fidelity-first raw routes
 # remain retry- and circuit-free. The default two half-open probes support two
 # concurrent Codex instances while keeping recovery pressure bounded. State

README.md

@@ -349,6 +349,12 @@ the proxy with `ANTHROPIC_BASE_URL`. See
 [protocol translation](docs/protocol-translation.md) for the coverage and
 limits.
 
+For Claude Haiku 5.5 in a coding harness, use the `/lean` base path
+(`/lean/v1/messages`, `/lean/v1/chat/completions`, `/lean/v1/responses`). It
+turns on prompt caching, clears old tool results server-side, condenses shell
+output and keeps every prompt under the 100,000-token long-prompt price. See
+[lean endpoint](docs/lean.md).
+
 ```bash
 # OpenAI-compatible endpoint
 curl -X POST "http://localhost:1400/openai/v1/chat/completions" \

app.py

@@ -10,6 +10,7 @@
 from env_loader import load_runtime_env
 from error_handlers import init_error_handlers
 from middleware.external_origin import TrustedExternalOriginMiddleware
+from middleware.lean_path import LeanPathMiddleware
 from route_helpers import (
     api_auth_required,
     apply_cors_headers,
@@ -30,6 +31,7 @@
 from routes.gateway_mcp import register_gateway_mcp_routes
 from routes.gateway_batches import register_gateway_batch_routes
 from routes.context_pages import register_context_page_routes
+from routes.lean import register_lean_routes
 from routes.optimized import register_optimized_routes
 from routes.proxy import register_proxy_routes
 from routes.status_page import register_status_routes
@@ -105,6 +107,8 @@ def create_app() -> Flask:
         # normalization and wins when Flask constructs public URLs.
         app.wsgi_app = TrustedExternalOriginMiddleware(app.wsgi_app)
         app.wsgi_app = ProxyFix(app.wsgi_app, x_proto=1, x_host=1)
+    if os.environ.get("LEAN_ENDPOINT_ENABLED", "").strip().lower() not in {"0", "false", "no", "off"}:
+        app.wsgi_app = LeanPathMiddleware(app.wsgi_app)
     app.secret_key = runtime_secrets["FLASK_SECRET_KEY"]
     app.config["JWT_SECRET"] = runtime_secrets["JWT_SECRET"]
 
@@ -146,6 +150,7 @@ def create_app() -> Flask:
     register_intelligence_media_routes(app, csrf, AuthService, ProxyService)
     register_free_routes(app, csrf, AuthService, MetricsService, ProxyService)
     register_optimized_routes(app, csrf, AuthService, MetricsService, ProxyService)
+    register_lean_routes(app)
     register_status_routes(app, csrf, AuthService, ProxyService)
     register_core_routes(app)
     register_dashboard_security_routes(app)

docs/lean.md

@@ -0,0 +1,177 @@
+# Lean endpoint
+
+`/lean/v1/messages`, `/lean/v1/chat/completions` and `/lean/v1/responses` are the
+unified routes with extra cost controls for models served by the Claude API
+(`anthropic:` models). They are written for coding harnesses on Claude Haiku 5.5,
+whose price depends on prompt length:
+
+| Haiku 5.5, per million tokens | Prompt of 100,000 tokens or fewer | Longer prompt |
+|---|---|---|
+| Input | $0.10 | $0.50 |
+| Cache read | $0.01 | $0.05 |
+| Cache write (5 minutes) | $0.125 | $0.625 |
+| Output | $0.50 | $2.50 |
+
+The prompt length counts every input token, cache reads and writes included, and
+a request one token over the limit pays the higher price on all of its tokens
+([pricing](https://platform.claude.com/docs/en/about-claude/pricing#long-context-pricing)).
+Lean mode keeps prompts cached and under that limit.
+
+Requests to other providers behave exactly as on `/v1` and carry
+`X-MultiLLM-Lean: inactive`. `/lean/v1/messages/count_tokens` and `/lean/v1/models`
+are the `/v1` routes unchanged, so a harness can use one base URL.
+
+## What lean mode does
+
+Every change depends only on the request itself, so each later request of a
+conversation repeats the earlier one's bytes up to its new turn. That keeps prompt
+caching working, and keeps Haiku 5.5's replayed thinking blocks valid: Haiku 5.5
+rejects a thinking block whose earlier conversation changed. Checked against the
+Claude API: a second identical request read its whole 2,006-token prefix from the
+cache, and a thinking block Haiku wrote through `/lean` was accepted when the next
+turn replayed it through `/lean`, but rejected when the same turn went to `/v1`
+with the raw shell output.
+
+| Stage | Default | What it does |
+|---|---|---|
+| Prompt caching | on | Adds automatic caching (`cache_control` at the top level) and a breakpoint after the system prompt, or after the last tool when there is none. Skipped when the caller placed its own `cache_control`, as Claude Code does. |
+| Tool result clearing | on, from 80,000 tokens | Adds `context_management` with `clear_tool_uses_20250919`: once the prompt passes the trigger, the API replaces old tool results with placeholders and keeps the 3 most recent, clearing at least 20,000 tokens at a time. The API applies it before the model reads the prompt, the harness keeps its full history, and it never invalidates thinking blocks. Skipped when the caller sends its own `context_management` or a compaction block. |
+| Shell output | on | Results of shell tools (`Bash`, `bash`, `shell`, `exec_command`, `run_shell_command` and similar) lose ANSI escapes and progress-bar redraws; runs of four or more identical lines become one line and a count, and runs of blank lines become one. Other tools' results, file reads included, keep their exact bytes, so an edit tool can still match them. |
+| Oversized results | 100,000 characters | A longer tool result keeps its first and last 50,000 characters with a marker in between. |
+| Prompt limit | 99,000 tokens | A prompt estimated near the limit is counted with `count_tokens`, using the key that will send it, which reports its size before and after context editing. Anthropic calls the count an estimate that can differ slightly, so the default sits 1,000 tokens under the 100,000-token price line. Above the limit the request is refused with the error the harness treats as a full context window, so it compacts and retries instead of paying the higher price. The pricing page does not say whether the size before or after clearing picks the price, so by default the limit applies to the size before clearing; set `LEAN_PRICE_LIMIT_BASIS=cleared` once your bill shows that cleared prompts are priced by their smaller size. |
+| Concise answers | off | Adds a short instruction to the system prompt: lead with the answer, no filler, code and commands exact. Turn it on with `LEAN_STYLE=concise` or per request with `X-MultiLLM-Lean-Style: concise` (`caveman` is accepted too). |
+
+Lean mode never changes the model, effort, thinking settings or `max_tokens`.
+
+### The refusal
+
+| Route | Response |
+|---|---|
+| `/lean/v1/messages` | 400, Anthropic envelope, `invalid_request_error`, `prompt is too long: N tokens > 99000 maximum`. Claude Code compacts on this message. |
+| `/lean/v1/chat/completions` | 400, `{"error": {"code": "context_length_exceeded", ...}}` with a message naming the maximum context length. |
+| `/lean/v1/responses` with `stream: true` | A `response.failed` event with `context_length_exceeded`, the form Codex reads. Without streaming, the Chat Completions body above. |
+
+When the prompt cannot be counted, the local estimate decides, and an estimate
+above the limit is refused. That happens for URL and file sources, which
+`count_tokens` rejects (lean estimates them at 1,600 tokens per image and 10,000
+per document); for a request that replays thinking blocks alongside a server tool
+such as web search, because `count_tokens` rejects server tools and dropping them
+would fail its thinking-block check; and when a count fails, for example when
+rate-limited. Such requests report `X-MultiLLM-Lean-Prompt-Count: estimate`, and
+the limit is then a safety net, not a guarantee: the estimate can be low for a
+URL document or an unusual tokenization. Without replayed thinking, server tools
+are left out of the count and added back at 1,000 tokens each. A prompt with a
+PDF is always counted, since its file size does not bound the text inside. A count
+that refuses the key itself (401, 402 or 403) does not decide: the request goes
+out so key rotation can rest that key and count again with the next one
+(`X-MultiLLM-Lean-Prompt-Count: skipped`). If compacting cannot bring a
+prompt under the limit, for example because the newest message alone is larger,
+the harness reports the error: Claude Code stops after three failed compactions.
+
+### Response headers
+
+| Header | Meaning |
+|---|---|
+| `X-MultiLLM-Lean` | `applied` for a Claude API request, `inactive` for other providers |
+| `X-MultiLLM-Lean-Rules` | Version of the rules above |
+| `X-MultiLLM-Lean-Stages` | Stages that changed the request, for example `cache,clear-tool-uses,condense-shell` |
+| `X-MultiLLM-Lean-Prompt-Tokens` | Prompt size the limit was checked against: counted by the API near the limit, otherwise a rough local estimate |
+| `X-MultiLLM-Lean-Prompt-Count` | `provider`, `estimate`, or `skipped` when the count refused the key |
+| `X-MultiLLM-Lean-Prompt-Tokens-After-Clearing` | Counted size after context editing, when it differs |
+| `X-MultiLLM-Lean-Condensed-Chars` | Characters removed from tool results |
+
+The API reports what it cleared in the response's `context_management` field and
+cache use in `usage.cache_read_input_tokens` and `usage.cache_creation_input_tokens`.
+
+## Harness setup
+
+Set each harness to compact before 100,000 tokens, so the refusal stays a safety
+net.
+
+### Claude Code
+
+```bash
+export ANTHROPIC_BASE_URL="$PROXY_BASE_URL/lean"
+export ANTHROPIC_API_KEY="$MULTILLM_API_KEY"
+export ANTHROPIC_MODEL="anthropic:claude-haiku-5-5"
+export ANTHROPIC_DEFAULT_HAIKU_MODEL="anthropic:claude-haiku-5-5"
+export CLAUDE_CODE_AUTO_COMPACT_WINDOW=100000
+claude
+```
+
+Claude Code places its own cache breakpoints, so lean mode leaves caching to it.
+
+### Codex
+
+```toml
+model = "anthropic:claude-haiku-5-5"
+model_provider = "multillm-lean"
+model_context_window = 100000
+model_auto_compact_token_limit = 85000
+
+[model_providers.multillm-lean]
+name = "MultiLLM lean"
+base_url = "https://multillm-proxy.cserules.workers.dev/lean/v1"
+env_key = "MULTILLM_API_KEY"
+wire_api = "responses"
+```
+
+### opencode
+
+```json
+{
+  "provider": {
+    "multillm-lean": {
+      "npm": "@ai-sdk/openai-compatible",
+      "options": {
+        "baseURL": "https://multillm-proxy.cserules.workers.dev/lean/v1",
+        "apiKey": "{env:MULTILLM_API_KEY}"
+      },
+      "models": {
+        "anthropic:claude-haiku-5-5": { "limit": { "context": 100000, "output": 32000 } }
+      }
+    }
+  }
+}
+```
+
+Chat Completions and Responses requests reach the Claude API translated, and the
+translation drops caller `cache_control`, so for Codex and opencode lean mode's
+caching is what makes cache reads happen at all.
+
+## Settings
+
+| Variable | Default | |
+|---|---|---|
+| `LEAN_ENDPOINT_ENABLED` | `true` | `false` stops serving `/lean` |
+| `LEAN_PROMPT_TOKEN_LIMIT` | `99000` | `0` turns the refusal off |
+| `LEAN_PRICE_LIMIT_BASIS` | `original` | `cleared` applies the limit to the prompt after context editing |
+| `LEAN_CLEAR_TRIGGER_TOKENS` | `80000` | `0` turns tool result clearing off |
+| `LEAN_CLEAR_KEEP_TOOL_USES` | `3` | Recent tool uses that are never cleared |
+| `LEAN_CLEAR_AT_LEAST_TOKENS` | `20000` | Minimum cleared per edit, so a cache rewrite is worth it |
+| `LEAN_CLEAR_EXCLUDE_TOOLS` | empty | Comma-separated tool names whose results are never cleared |
+| `LEAN_CACHE_TTL` | `5m` | `1h` for sessions with long pauses; 1-hour writes cost 2x input |
+| `LEAN_CONDENSE_SHELL_OUTPUT` | `true` | |
+| `LEAN_TOOL_RESULT_MAX_CHARS` | `100000` | `0` turns clipping off |
+| `LEAN_STYLE` | `none` | `concise` adds the concise-answer instruction |
+
+Keep a conversation on one base URL and its settings fixed. Lean mode sends shell
+output condensed, so moving a conversation between `/v1` and `/lean`, or changing
+`LEAN_CONDENSE_SHELL_OUTPUT`, `LEAN_TOOL_RESULT_MAX_CHARS` or `LEAN_STYLE` partway
+through, changes bytes the model already saw. On accounts created on or after
+31 August 2026 the next request that replays a thinking block then fails with
+``Invalid `signature` in `thinking` block. The block is bound to a different
+conversation``; start a new conversation. The cache and clearing settings change
+only cost: one cache rewrite.
+
+## Notes
+
+- Cleared tool results are gone for the model, not for the harness: if the model
+  needs one again it reruns the tool.
+- The Claude API's prompt cache belongs to the account. The key pool keeps using
+  the first key until it is refused ([key failover](anthropic.md#spare-keys-and-failover)); a conversation
+  that moves to the spare key starts with a cold cache, and its earlier thinking
+  blocks are dropped rather than rejected.
+- Research behind these choices, including what Claude Code, Codex, opencode,
+  Cline and Gemini CLI do, is summarised in the pull request that added this
+  page.

middleware/lean_path.py

@@ -0,0 +1,43 @@
+"""Serve `/lean/v1/...` from the unified `/v1/...` routes in lean mode."""
+
+from collections.abc import Callable, Iterable
+from typing import Any
+
+WSGIStartResponse = Callable[..., Any]
+WSGIApplication = Callable[[dict[str, Any], WSGIStartResponse], Iterable[bytes]]
+
+LEAN_ENVIRON_KEY = "multillm.lean"
+LEAN_PREFIX = "/lean"
+LEAN_PATHS = frozenset(
+    {
+        "/v1/chat/completions",
+        "/v1/messages",
+        "/v1/messages/count_tokens",
+        "/v1/models",
+        "/v1/responses",
+    }
+)
+
+
+class LeanPathMiddleware:
+    """Rewrite a lean path to its unified route and mark the request as lean.
+
+    Rewriting before Flask routes the request keeps every path-based rule (rate
+    limits, body limits, error envelopes, cache affinity) identical to `/v1`.
+    The mark lives in the WSGI environ, which a client cannot set.
+    """
+
+    def __init__(self, application: WSGIApplication) -> None:
+        self.application = application
+
+    def __call__(
+        self,
+        environ: dict[str, Any],
+        start_response: WSGIStartResponse,
+    ) -> Iterable[bytes]:
+        environ.pop(LEAN_ENVIRON_KEY, None)
+        path = environ.get("PATH_INFO", "")
+        if path.startswith(f"{LEAN_PREFIX}/") and path[len(LEAN_PREFIX):] in LEAN_PATHS:
+            environ["PATH_INFO"] = path[len(LEAN_PREFIX):]
+            environ[LEAN_ENVIRON_KEY] = True
+        return self.application(environ, start_response)

routes/lean.py

@@ -0,0 +1,55 @@
+"""Lean mode responses: report headers and the protocol's own "prompt is too long" error."""
+
+from __future__ import annotations
+
+import json
+
+from flask import Response, jsonify, request
+
+from error_handlers import get_request_id
+from services.lean_mode import LeanPromptTooLong, lean_response_headers
+from services.protocol_translation import anthropic_error_body
+
+
+def prompt_too_long_response(error: LeanPromptTooLong):
+    """Refuse in the shape each harness treats as a context overflow, so it compacts.
+
+    Claude Code compacts on Anthropic's "prompt is too long" error. OpenAI-style
+    clients match `context_length_exceeded`; Codex reads it from a streamed
+    `response.failed` event.
+    """
+    request_id = get_request_id()
+    if request.path.startswith("/v1/messages"):
+        return jsonify(anthropic_error_body(400, str(error), "invalid_request_error", request_id)), 400
+    message = (
+        f"This model's maximum context length is {error.limit} tokens. However, your "
+        f"messages resulted in {error.tokens} tokens ({error}). Compact the "
+        "conversation and try again."
+    )
+    payload = request.get_json(silent=True)
+    if request.path == "/v1/responses" and isinstance(payload, dict) and payload.get("stream"):
+        event = {
+            "type": "response.failed",
+            "sequence_number": 0,
+            "response": {
+                "object": "response",
+                "status": "failed",
+                "error": {"code": "context_length_exceeded", "message": message},
+            },
+        }
+        return Response(
+            f"event: response.failed\ndata: {json.dumps(event)}\n\n",
+            mimetype="text/event-stream",
+        )
+    error_body = {
+        "message": message,
+        "type": "invalid_request_error",
+        "param": "messages",
+        "code": "context_length_exceeded",
+    }
+    return jsonify({"error": error_body, "request_id": request_id}), 400
+
+
+def register_lean_routes(app) -> None:
+    app.after_request(lean_response_headers)
+    app.register_error_handler(LeanPromptTooLong, prompt_too_long_response)

routes/unified.py

@@ -68,6 +68,7 @@
 from services.managed_turn import bounded_managed_timeout as bounded_timeout
 from routes.cascades import dispatch_unified_cascade, register_cascade_admin_routes, validate_cascade_target
 from services.context_optimizer import ContextOptimizationResult
+from services.lean_mode import check_lean_prompt, merge_beta_header, prepare_lean_request
 from services.model_registry import ModelRegistry
 from services.model_cooldown import ModelCooldownCapacity, ModelCooldownExhausted
 from routes.provider_credentials import (
@@ -307,16 +308,23 @@ def _send_native_request(
 ):
     """Send one body to a provider's native Responses or Messages endpoint."""
     from services.managed_turn import prepare_dispatch_kwargs
-    raw_body = json.dumps(body).encode("utf-8")
+    lean = prepare_lean_request(
+        body, provider=provider, endpoint=endpoint, request_headers=headers_source
+    )
+    raw_body = json.dumps(lean.body).encode("utf-8")
 
     def send_request(token: str):
+        check_lean_prompt(
+            lean, token, request_headers=headers_source, base_urls=app.config["API_BASE_URLS"]
+        )
         headers = proxy_service_cls.prepare_headers(
             headers_source,
             provider,
             token,
             upstream_path=endpoint,
         )
         _merge_request_headers(headers, cache_headers)
+        merge_beta_header(headers, lean.betas)
         default_url = f"{app.config['API_BASE_URLS'][provider].rstrip('/')}/{endpoint}"
         request_kwargs = {
             "method": "POST",