From b82f8a095823b806b11556ee5c612cffdd9cd7f6 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Thu, 22 Oct 2026 23:27:57 -0700 Subject: [PATCH 1/3] Price duplicated model ids by owner, never by file order A model id listed by several models.dev providers resolves to the owning lab's own listing (named by canonical_model_id), else a first-party vendor's in fixed precedence, else the tariff most paid listings agree on, ties broken by provider id. Retired first-party ids stay unpriced. PricingTable also keeps GitHub Copilot's own tariffs, and every cost view looks rates up with the turn's source so Copilot CLI turns price at Copilot's tariff where it lists the model. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + .../relayburn-sdk/src/analyze/cache_expiry.rs | 5 +- crates/relayburn-sdk/src/analyze/claude_md.rs | 2 +- .../src/analyze/context_delta.rs | 4 +- crates/relayburn-sdk/src/analyze/cost.rs | 75 ++- .../src/analyze/ghost_surface_inputs.rs | 26 +- crates/relayburn-sdk/src/analyze/hotspots.rs | 2 +- .../src/analyze/patterns/compaction.rs | 5 +- crates/relayburn-sdk/src/analyze/pricing.rs | 537 +++++++++++++++--- .../src/analyze/tool_call_patterns.rs | 6 +- .../src/analyze/tool_output_bloat.rs | 6 +- .../src/ingest/ingest_tests/copilot.rs | 7 +- .../reasoning/routine_tests.rs | 2 +- 13 files changed, 565 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 310bf617..2ac64506 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Cross-package release notes for relayburn. Package changelogs contain package-le - A Claude assistant turn still being written (no `stop_reason` yet) is billed once it settles, not early. - A Claude subagent turn that follows a tool result inside its sidechain keeps the subagent's agent id instead of starting a new subagent. - Pricing bills Claude 1-hour cache writes at 2x the input tariff on every host (Anthropic, Vertex AI, Bedrock; 5-minute writes stay at the `cache_write` tariff), so `summary`, `hotspots`, `analyze`, and every cost report stop undercounting sessions that use 1-hour prompt caching. `ModelCost` / `ModelCostTier` gain `cache_write_1h`, and a ledger-home `models.dev.json` entry can set `cache_write_1h` explicitly; non-Claude models keep a single cache-write tariff. +- Pricing resolves a model id that several providers list without consulting file order: the owning lab's own tariff (named by models.dev `canonical_model_id`, e.g. Moonshot for `kimi-*`), else a first-party vendor's, else the tariff most paid reseller listings agree on. `kimi-k3` stops billing cache reads at the input rate, and 409 of 3,796 bundled model ids move to their owner's or the consensus tariff. Copilot CLI turns price at GitHub Copilot's own tariff wherever it lists the model. - Turns whose usage was measured as zero price at $0 regardless of model, so Claude Code's `` messages no longer list as unpriced in `summary` or strip the dollar estimate from every hotspots finding in their session; unmeasured usage stays unpriced. `burn compare` shows cost per turn only for fully priced cells. ### Analysis and quality diff --git a/crates/relayburn-sdk/src/analyze/cache_expiry.rs b/crates/relayburn-sdk/src/analyze/cache_expiry.rs index 1331adbf..a547560f 100644 --- a/crates/relayburn-sdk/src/analyze/cache_expiry.rs +++ b/crates/relayburn-sdk/src/analyze/cache_expiry.rs @@ -303,7 +303,10 @@ fn recreated_prefix(prev: &Usage, cur: &Usage) -> Option { /// Extra USD `turn` paid to re-write `recreated` tokens versus reading them /// from a warm cache. `None` when the model is unpriced. fn expiry_penalty(turn: &TurnRecord, recreated: u64, pricing: &PricingTable) -> Option { - let rate = effective_model_rate(&turn.usage, lookup_model_rate(&turn.model, pricing)?); + let rate = effective_model_rate( + &turn.usage, + lookup_model_rate(&turn.model, Some(turn.source), pricing)?, + ); let created = turn.usage.cache_create_5m + turn.usage.cache_create_1h; let recreated_write = rate.cache_create_cost(&turn.usage) * recreated as f64 / created as f64; let warm_read = recreated as f64 / PER_MILLION * rate.cache_read; diff --git a/crates/relayburn-sdk/src/analyze/claude_md.rs b/crates/relayburn-sdk/src/analyze/claude_md.rs index c3053fa3..be155f4c 100644 --- a/crates/relayburn-sdk/src/analyze/claude_md.rs +++ b/crates/relayburn-sdk/src/analyze/claude_md.rs @@ -338,7 +338,7 @@ pub(crate) fn attribute_claude_md_refs( // or not; only priced turns add cost. let rides = t.usage.cache_read >= total_tokens; riding_turns += u64::from(rides); - let Some(rate) = lookup_model_rate(&t.model, pricing) else { + let Some(rate) = lookup_model_rate(&t.model, Some(t.source), pricing) else { continue; }; *model_counts.entry(t.model.clone()).or_insert(0) += 1; diff --git a/crates/relayburn-sdk/src/analyze/context_delta.rs b/crates/relayburn-sdk/src/analyze/context_delta.rs index 7843f3db..6f671d3e 100644 --- a/crates/relayburn-sdk/src/analyze/context_delta.rs +++ b/crates/relayburn-sdk/src/analyze/context_delta.rs @@ -473,7 +473,9 @@ fn attributed_cost(delta_tokens: i64, model: &str, pricing: &PricingTable) -> f6 if delta_tokens <= 0 { return 0.0; } - let Some(rate) = crate::analyze::cost::lookup_model_rate(model, pricing) else { + // Span trees carry no harness; their Claude and Codex turns take the + // resolved tariff either way. + let Some(rate) = crate::analyze::cost::lookup_model_rate(model, None, pricing) else { return 0.0; }; // Charge at cache_read because cache_read is what every *future* diff --git a/crates/relayburn-sdk/src/analyze/cost.rs b/crates/relayburn-sdk/src/analyze/cost.rs index 408857a9..0b2d7345 100644 --- a/crates/relayburn-sdk/src/analyze/cost.rs +++ b/crates/relayburn-sdk/src/analyze/cost.rs @@ -12,7 +12,9 @@ use serde::{Deserialize, Serialize}; use crate::reader::{SourceKind, TurnRecord, Usage, UsageGranularity}; -use crate::analyze::pricing::{ModelCost, ModelCostTier, PricingTable, ReasoningMode}; +use crate::analyze::pricing::{ + billing_provider, ModelCost, ModelCostTier, PricingTable, ReasoningMode, +}; use crate::analyze::provider_reattribution::{resolve_provider, strip_provider_prefix}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -43,6 +45,9 @@ pub struct CostForUsageOptions { /// aggregate), so no request is known to cross a context tier: the /// model's base rates apply. pub base_tier: bool, + /// Harness the usage was read from; selects the tariff of a provider + /// that bills that harness's sessions (see [`billing_provider`]). + pub source: Option, } /// Tokens-per-million divisor shared by every per-token cost calc in the @@ -73,7 +78,7 @@ pub(crate) fn cost_for_usage( pricing: &PricingTable, options: CostForUsageOptions, ) -> Option { - let rate = lookup_model_rate(model, pricing)?; + let rate = lookup_model_rate(model, options.source, pricing)?; let tier = if options.base_tier { None } else { @@ -118,6 +123,7 @@ pub fn cost_for_turn(turn: &TurnRecord, pricing: &PricingTable) -> Option bool { reasoning_mode_for_source(source) == Some(ReasoningMode::IncludedInOutput) } -/// Shared lookup: direct match → synthetic reattribution → generic -/// `provider/model` strip. Used by `cost_for_usage` and (later) by the -/// hotspots / claude-md attribution paths so synthetic-routed turns price -/// consistently across views. -pub fn lookup_model_rate<'a>(model: &str, pricing: &'a PricingTable) -> Option<&'a ModelCost> { - if let Some(direct) = pricing.get(model) { +/// Shared lookup: direct match → alias → synthetic reattribution → generic +/// `provider/model` strip, each step preferring the tariff of the provider +/// that bills `source`'s sessions. Every cost view prices through here so a +/// turn costs the same in all of them. +pub fn lookup_model_rate<'a>( + model: &str, + source: Option, + pricing: &'a PricingTable, +) -> Option<&'a ModelCost> { + let billed_by = source.and_then(billing_provider); + let rate = |id: &str| pricing.rate(id, billed_by); + if let Some(direct) = rate(model) { return Some(direct); } - if let Some(alias) = pricing_model_alias(model) { - if let Some(via) = pricing.get(alias) { - return Some(via); - } + if let Some(via) = pricing_model_alias(model).and_then(rate) { + return Some(via); } let reattributed = resolve_provider(model); if reattributed.normalized_model != model { - if let Some(via) = pricing.get(&reattributed.normalized_model) { + if let Some(via) = rate(&reattributed.normalized_model) { return Some(via); } } let stripped = strip_provider_prefix(model); if stripped != model { - if let Some(via) = pricing.get(stripped) { + if let Some(via) = rate(stripped) { return Some(via); } } @@ -682,6 +692,33 @@ mod tests { assert_eq!(skipped.reasoning, 0.0); } + #[test] + fn copilot_turns_price_at_copilots_tariff() { + // A cache-read-heavy kimi-k3 Copilot turn bills its cache reads at + // Copilot's cache-read tariff, well under its input tariff. + let p = load_builtin_pricing(); + let usage = Usage { + input: 20_000, + output: 5_000, + reasoning: 0, + cache_read: 2_000_000, + cache_create_5m: 0, + cache_create_1h: 0, + }; + let copilot = p.rate("kimi-k3", Some("github-copilot")).unwrap(); + assert!(copilot.cache_read < copilot.input); + let expected = (20_000.0 * copilot.input + + 5_000.0 * copilot.output + + 2_000_000.0 * copilot.cache_read) + / PER_MILLION; + let c = cost_for_turn(&turn("kimi-k3", usage, SourceKind::CopilotCli), &p).unwrap(); + assert!( + (c.total - expected).abs() < 1e-12, + "{} vs {expected}", + c.total + ); + } + #[test] fn returns_none_for_unknown_model() { let p = load_builtin_pricing(); @@ -723,7 +760,8 @@ mod tests { context_tiers: Vec::new(), }, ); - let rate = lookup_model_rate("synthetic/qwen3-coder", &p).expect("synthetic prefix routes"); + let rate = + lookup_model_rate("synthetic/qwen3-coder", None, &p).expect("synthetic prefix routes"); assert_eq!(rate.input, 1.0); assert_eq!(rate.output, 2.0); } @@ -744,7 +782,8 @@ mod tests { context_tiers: Vec::new(), }, ); - let rate = lookup_model_rate("codex-auto-review", &p).expect("auto-review alias routes"); + let rate = + lookup_model_rate("codex-auto-review", None, &p).expect("auto-review alias routes"); assert_eq!(rate.input, 1.75); assert_eq!(rate.output, 14.0); assert_eq!(rate.cache_read, 0.175); @@ -779,8 +818,8 @@ mod tests { context_tiers: Vec::new(), }, ); - let rate = - lookup_model_rate("codex-auto-review", &p).expect("direct auto-review price wins"); + let rate = lookup_model_rate("codex-auto-review", None, &p) + .expect("direct auto-review price wins"); assert_eq!(rate.input, 9.0); assert_eq!(rate.output, 10.0); assert_eq!(rate.cache_read, 1.0); diff --git a/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs b/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs index a9706559..0bd2cec4 100644 --- a/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs +++ b/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs @@ -18,7 +18,7 @@ use std::collections::{HashMap, HashSet}; use crate::reader::{SourceKind, TurnRecord}; use crate::analyze::ghost_surface::GhostSurfaceInputs; -use crate::analyze::pricing::PricingTable; +use crate::analyze::pricing::{billing_provider, PricingTable}; /// Per-source set of *observed invocation names* in this slice. Each turn /// contributes its tool-call names, any `skillName` set on a tool call, and @@ -66,29 +66,27 @@ pub fn build_session_count_by_source(turns: &[TurnRecord]) -> HashMap f64 { let mut counts: HashMap<&str, u64> = HashMap::new(); - let mut order: Vec<&str> = Vec::new(); + let mut order: Vec<(&str, SourceKind)> = Vec::new(); for t in turns { let m = t.model.as_str(); if !counts.contains_key(m) { - order.push(m); + order.push((m, t.source)); } *counts.entry(m).or_insert(0) += 1; } - let mut best_model: Option<&str> = None; + let mut best_rate: Option = None; let mut best_count: i64 = -1; // Iterate in first-seen order so ties go to the earliest-seen model // (matches the TS `Map` iteration order). - for m in order { + for (m, source) in order { let c = counts[m] as i64; - if c > best_count && pricing.contains_key(m) { - best_model = Some(m); + let rate = pricing.rate(m, billing_provider(source)); + if let Some(rate) = rate.filter(|_| c > best_count) { + best_rate = Some(rate.cache_read); best_count = c; } } - match best_model { - Some(m) => pricing[m].cache_read / 1_000_000.0, - None => 0.0, - } + best_rate.map_or(0.0, |cache_read| cache_read / 1_000_000.0) } /// Assemble a `GhostSurfaceInputs` from a slice of turns plus a pricing @@ -230,7 +228,7 @@ mod tests { #[test] fn picks_cache_read_rate_of_most_used_priced_model() { - let mut pricing: PricingTable = HashMap::new(); + let mut pricing = PricingTable::new(); pricing.insert( "claude-sonnet-4-6".to_string(), ModelCost { @@ -286,14 +284,14 @@ mod tests { #[test] fn picks_zero_when_no_priced_model() { - let pricing: PricingTable = HashMap::new(); + let pricing = PricingTable::new(); let turns = vec![make_turn(SourceKind::Codex, "s1", "unknown", vec![], None)]; assert_eq!(pick_representative_cache_read_rate(&turns, &pricing), 0.0); } #[test] fn build_inputs_drops_empty_user_turn_text() { - let pricing: PricingTable = HashMap::new(); + let pricing = PricingTable::new(); let inputs = build_ghost_surface_inputs(&[], &pricing, Some(HashMap::new())); assert!(inputs.user_turn_text_by_session.is_none()); } diff --git a/crates/relayburn-sdk/src/analyze/hotspots.rs b/crates/relayburn-sdk/src/analyze/hotspots.rs index c048399d..85f55667 100644 --- a/crates/relayburn-sdk/src/analyze/hotspots.rs +++ b/crates/relayburn-sdk/src/analyze/hotspots.rs @@ -456,7 +456,7 @@ fn attribute_session( let mut grand_total = 0.0_f64; for &turn in turns { - let turn_rate = lookup_model_rate(&turn.model, pricing) + let turn_rate = lookup_model_rate(&turn.model, Some(turn.source), pricing) .map(|rate| effective_model_rate(&turn.usage, rate)); // Accumulate the per-turn grand total in this same pass. Routes diff --git a/crates/relayburn-sdk/src/analyze/patterns/compaction.rs b/crates/relayburn-sdk/src/analyze/patterns/compaction.rs index 8275641f..73f7fb88 100644 --- a/crates/relayburn-sdk/src/analyze/patterns/compaction.rs +++ b/crates/relayburn-sdk/src/analyze/patterns/compaction.rs @@ -71,7 +71,10 @@ pub(super) fn detect_compaction_losses( &usage, &preceding.model, pricing, - CostForUsageOptions::default(), + CostForUsageOptions { + source: Some(preceding.source), + ..CostForUsageOptions::default() + }, ) { cache_lost_cost = priced.total; } diff --git a/crates/relayburn-sdk/src/analyze/pricing.rs b/crates/relayburn-sdk/src/analyze/pricing.rs index 8d2e8c63..5c5fea10 100644 --- a/crates/relayburn-sdk/src/analyze/pricing.rs +++ b/crates/relayburn-sdk/src/analyze/pricing.rs @@ -11,9 +11,10 @@ use std::io; use std::path::Path; use std::sync::LazyLock; -use indexmap::IndexMap; use serde::{Deserialize, Serialize}; +use crate::reader::SourceKind; + /// How a model's reasoning tokens should be priced. /// /// - `IncludedInOutput`: the harness/source already counts reasoning tokens @@ -66,12 +67,93 @@ pub struct ModelCostTier { pub reasoning: Option, } -pub(crate) type PricingTable = HashMap; +/// Per-model tariffs resolved from a models.dev-shaped snapshot, plus the +/// tariffs of providers that bill a harness's sessions themselves. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct PricingTable { + /// One tariff per model id, chosen by [`resolve_tariff`]. + models: HashMap, + /// provider id → model id → that provider's own tariff, kept only for + /// [`BILLING_PROVIDERS`]. + billed: HashMap>, +} + +impl PricingTable { + pub fn new() -> Self { + Self::default() + } + + /// The resolved tariff for `model`, independent of who served it. + pub fn get(&self, model: &str) -> Option<&ModelCost> { + self.models.get(model) + } + + /// The tariff `billing_provider` charges for `model` when it lists one, + /// else the resolved tariff. + pub fn rate(&self, model: &str, billing_provider: Option<&str>) -> Option<&ModelCost> { + billing_provider + .and_then(|provider| self.billed.get(provider)) + .and_then(|models| models.get(model)) + .or_else(|| self.get(model)) + } + + pub fn contains_key(&self, model: &str) -> bool { + self.models.contains_key(model) + } + + pub fn iter(&self) -> impl Iterator { + self.models.iter() + } + + pub fn values(&self) -> impl Iterator { + self.models.values() + } + + /// Set `model`'s tariff for every session, whoever bills it. + pub fn insert(&mut self, model: String, cost: ModelCost) { + for models in self.billed.values_mut() { + models.remove(&model); + } + self.models.insert(model, cost); + } + + /// Shadow this table with `overlay`: its tariffs replace these for every + /// model it prices, its billing-provider tariffs included. + fn merge(&mut self, overlay: PricingTable) { + for (model, cost) in overlay.models { + self.insert(model, cost); + } + for (provider, models) in overlay.billed { + self.billed.entry(provider).or_default().extend(models); + } + } +} + +const GITHUB_COPILOT: &str = "github-copilot"; + +/// Providers whose own tariff bills a session whenever burn reads that +/// session from their harness; see [`billing_provider`]. +const BILLING_PROVIDERS: &[&str] = &[GITHUB_COPILOT]; + +/// The provider that bills every model call of a `source` session, when the +/// harness fixes it. Copilot CLI routes every call through GitHub Copilot. +/// Other harnesses can run against several providers under the same model +/// id, so their turns take the resolved tariff. +pub(crate) fn billing_provider(source: SourceKind) -> Option<&'static str> { + match source { + SourceKind::CopilotCli => Some(GITHUB_COPILOT), + _ => None, + } +} #[derive(Debug, Default, Deserialize)] struct ModelsDevModel { #[serde(default)] cost: Option, + /// `owner/model` naming the lab that publishes the model, e.g. + /// `moonshotai/kimi-k3` on every reseller's `kimi-k3`. + #[serde(default)] + canonical_model_id: Option, } #[derive(Debug, Default, Deserialize)] @@ -120,17 +202,13 @@ struct ModelsDevTierCondition { #[derive(Debug, Default, Deserialize)] struct ModelsDevProvider { - // `IndexMap` preserves JSON insertion order so iteration over the model - // map matches the TS `Object.entries` walk; this is what makes - // duplicate-id resolution deterministic (last entry in file order wins). #[serde(default)] - models: Option>, + models: Option>, } -// Providers iterate in file order so duplicate non-primary model ids retain -// deterministic last-wins behavior. Primary vendor entries are protected -// from later reseller copies in `flatten`. -type ModelsDevRoot = IndexMap; +/// Provider order in the file is not a pricing contract: models.dev adds +/// provider blocks anywhere, so [`resolve_tariff`] never consults it. +type ModelsDevRoot = HashMap; /// models.dev publishes a single `cache_write` tariff, which for Claude is /// the 5-minute-TTL write (1.25x input). Claude prompt caching bills @@ -158,9 +236,9 @@ fn cache_write_1h_rate(model_id: &str, explicit: Option, input: f64, cache_ }) } -/// Primary model vendors whose own tariff should win over reseller/router -/// copies with the same bare model id. models.dev regularly adds new provider -/// blocks and their order is not a pricing-precedence contract. +/// First-party vendors, in precedence order, whose listing of a model id +/// prices it when no entry for that id names an owner that lists it +/// (`canonical_model_id` is missing on older first-party models). const PRIMARY_PRICING_PROVIDERS: &[&str] = &["anthropic", "openai", "google", "google-vertex", "xai"]; @@ -196,16 +274,14 @@ pub fn load_builtin_pricing() -> PricingTable { /// Load pricing, optionally merging an override file over the builtin. When /// `override_path` is `Some` and the file exists + parses, its entries shadow -/// builtin entries on key collision (TS `{ ...builtin, ...user }` semantics). -/// Any read or parse error on the override silently falls back to the builtin -/// — matches the TS `try/catch` in `loadPricing`. +/// builtin entries on key collision, billing-provider tariffs included (see +/// [`PricingTable::merge`]). Any read or parse error on the override silently +/// falls back to the builtin. pub fn load_pricing(override_path: Option<&Path>) -> PricingTable { let mut table = load_builtin_pricing(); if let Some(path) = override_path { if let Ok(user) = load_from_file(path) { - for (k, v) in user { - table.insert(k, v); - } + table.merge(user); } } table @@ -228,53 +304,149 @@ fn parse_pricing_with_protected_models( Ok(flatten(&parsed, protected_models)) } -/// Flatten a nested `provider → model → cost` map into the flat -/// `model_id → ModelCost` table burn uses for lookup. Skips entries that lack -/// either `input` or `output` — matches the TS guard so we don't surface -/// half-priced models. +/// One provider's priced listing of a model id. +struct Listing<'a> { + provider: &'a str, + cost: ModelCost, +} + +/// Flatten a nested `provider → model → cost` map into the per-model table +/// burn uses for lookup, one tariff per model id chosen by +/// [`resolve_tariff`]. Entries lacking `input` or `output` are skipped so no +/// half-priced model surfaces. fn flatten(root: &ModelsDevRoot, protected_models: &HashSet) -> PricingTable { - let mut out = PricingTable::new(); - let mut primary_models: HashSet = HashSet::new(); + let mut table = PricingTable::new(); + let mut listings: HashMap<&str, Vec>> = HashMap::new(); + let mut owner_votes: HashMap<&str, HashMap<&str, usize>> = HashMap::new(); for (provider_id, provider) in root { let Some(models) = provider.models.as_ref() else { continue; }; for (id, model) in models { - let primary_provider = PRIMARY_PRICING_PROVIDERS.contains(&provider_id.as_str()); - if !primary_provider && (primary_models.contains(id) || protected_models.contains(id)) { - continue; + if let Some((owner, _)) = model + .canonical_model_id + .as_deref() + .and_then(|canonical| canonical.split_once('/')) + { + *owner_votes.entry(id).or_default().entry(owner).or_default() += 1; } - let Some(cost) = model.cost.as_ref() else { + let Some(cost) = model_cost(id, model) else { continue; }; - let (Some(input), Some(output)) = (cost.input, cost.output) else { - continue; - }; - let has_reasoning = cost.reasoning.is_some(); - // Models that don't publish a cache-write rate get billed at the - // input rate for cache creation. - let cache_write = cost.cache_write.unwrap_or(input); - let entry = ModelCost { - input, - output, - cache_read: cost.cache_read.unwrap_or(0.0), - cache_write, - cache_write_1h: cache_write_1h_rate(id, cost.cache_write_1h, input, cache_write), - reasoning: cost.reasoning, - reasoning_mode: if has_reasoning { - ReasoningMode::Separate - } else { - ReasoningMode::SameAsOutput - }, - context_tiers: context_tiers(id, cost), - }; - out.insert(id.clone(), entry); - if primary_provider { - primary_models.insert(id.clone()); + if BILLING_PROVIDERS.contains(&provider_id.as_str()) { + table + .billed + .entry(provider_id.clone()) + .or_default() + .insert(id.clone(), cost.clone()); + } + listings.entry(id).or_default().push(Listing { + provider: provider_id, + cost, + }); + } + } + for (id, listings) in listings { + let owners = ranked_owners(owner_votes.remove(id).unwrap_or_default()); + if let Some(cost) = resolve_tariff(&listings, &owners, protected_models.contains(id)) { + table.models.insert(id.to_string(), cost.clone()); + } + } + table +} + +/// Owners named by a model id's `canonical_model_id`s, most-named first, ties +/// in name order. +fn ranked_owners(votes: HashMap<&str, usize>) -> Vec<&str> { + let mut owners: Vec<(&str, usize)> = votes.into_iter().collect(); + owners.sort_by(|(a, a_votes), (b, b_votes)| b_votes.cmp(a_votes).then_with(|| a.cmp(b))); + owners.into_iter().map(|(owner, _)| owner).collect() +} + +/// Choose a model id's tariff from its priced `listings`: +/// +/// 1. the owner's own listing, trying `owners` in rank order; +/// 2. else the listing of the first [`PRIMARY_PRICING_PROVIDERS`] vendor +/// that has one; +/// 3. else nothing when the id was ever first-party (`protected`), so a +/// retired first-party model is unpriced rather than inheriting a +/// reseller tariff; +/// 4. else the tariff the most paid listings agree on, ties going to the +/// tariff whose providers include the alphabetically first provider id. +/// A listing free on both input and output is a free tier or plan, not a +/// token tariff, so it prices the model only when every listing is free. +fn resolve_tariff<'a>( + listings: &'a [Listing<'_>], + owners: &[&str], + protected: bool, +) -> Option<&'a ModelCost> { + let listed_by = |provider: &str| { + listings + .iter() + .find(|listing| listing.provider == provider) + .map(|listing| &listing.cost) + }; + if let Some(cost) = owners.iter().find_map(|owner| listed_by(owner)) { + return Some(cost); + } + if let Some(cost) = PRIMARY_PRICING_PROVIDERS + .iter() + .find_map(|vendor| listed_by(vendor)) + { + return Some(cost); + } + if protected { + return None; + } + consensus_tariff(listings) +} + +fn consensus_tariff<'a>(listings: &'a [Listing<'_>]) -> Option<&'a ModelCost> { + let free = |listing: &&Listing<'_>| listing.cost.input == 0.0 && listing.cost.output == 0.0; + let all_free = listings.iter().all(|listing| free(&listing)); + // (tariff, listings agreeing, alphabetically first provider among them) + let mut groups: Vec<(&ModelCost, usize, &str)> = Vec::new(); + for listing in listings.iter().filter(|listing| all_free || !free(listing)) { + match groups + .iter_mut() + .find(|(cost, _, _)| **cost == listing.cost) + { + Some((_, count, first)) => { + *count += 1; + *first = (*first).min(listing.provider); } + None => groups.push((&listing.cost, 1, listing.provider)), } } - out + groups + .into_iter() + .max_by(|(_, a_count, a_first), (_, b_count, b_first)| { + a_count.cmp(b_count).then_with(|| b_first.cmp(a_first)) + }) + .map(|(cost, _, _)| cost) +} + +/// A listing's tariff, or `None` when it lacks `input` or `output`. +fn model_cost(id: &str, model: &ModelsDevModel) -> Option { + let cost = model.cost.as_ref()?; + let (input, output) = (cost.input?, cost.output?); + // Models that don't publish a cache-write rate get billed at the input + // rate for cache creation. + let cache_write = cost.cache_write.unwrap_or(input); + Some(ModelCost { + input, + output, + cache_read: cost.cache_read.unwrap_or(0.0), + cache_write, + cache_write_1h: cache_write_1h_rate(id, cost.cache_write_1h, input, cache_write), + reasoning: cost.reasoning, + reasoning_mode: if cost.reasoning.is_some() { + ReasoningMode::Separate + } else { + ReasoningMode::SameAsOutput + }, + context_tiers: context_tiers(id, cost), + }) } fn context_tiers(model_id: &str, cost: &ModelsDevCost) -> Vec { @@ -609,27 +781,252 @@ mod tests { assert_eq!(gpt.cache_write_1h, gpt.cache_write); } + /// `id`'s tariff as `provider` lists it in the bundled snapshot. + pub(crate) fn builtin_listing(provider: &str, id: &str) -> ModelCost { + let root: ModelsDevRoot = serde_json::from_str(BUILTIN_PRICING_JSON).unwrap(); + let model = &root[provider].models.as_ref().unwrap()[id]; + model_cost(id, model).unwrap_or_else(|| panic!("{provider}/{id} is priced")) + } + + /// The same providers in both file orders. + fn both_orders(blocks: &[&str]) -> [String; 2] { + let forward = blocks.join(","); + let reverse = blocks.iter().rev().copied().collect::>().join(","); + [format!("{{{forward}}}"), format!("{{{reverse}}}")] + } + #[test] - fn flatten_resolves_duplicate_model_ids_with_last_wins_file_order() { - // `models.dev.json` lists ~1900 model IDs that appear under multiple - // providers (e.g. `claude-sonnet-4-6` under both `anthropic` and - // `nano-gpt`). The TS path iterates `Object.values(root)` / - // `Object.entries(models)` in insertion order and lets the last - // assignment to `out[id]` win. We must do the same so duplicate-id - // pricing is deterministic across runs. - let raw = r#"{ - "first": { "models": { "shared": { "cost": { "input": 1, "output": 2 } } } }, - "second": { "models": { "shared": { "cost": { "input": 9, "output": 8 } } } } + fn flatten_prices_a_model_at_its_owners_tariff_in_any_file_order() { + // The kimi-k3 shape: the owner, a reseller that agrees with it, and + // a later reseller billing cache reads at the input rate. + for raw in both_orders(&[ + r#""moonshotai": { "models": { "kimi-x": { "canonical_model_id": "moonshotai/kimi-x", + "cost": { "input": 3, "output": 15, "cache_read": 0.3, "cache_write": 3 } } } }"#, + r#""reseller-a": { "models": { "kimi-x": { "canonical_model_id": "moonshotai/kimi-x", + "cost": { "input": 3, "output": 15, "cache_read": 0.3 } } } }"#, + r#""reseller-b": { "models": { "kimi-x": { "canonical_model_id": "moonshotai/kimi-x", + "cost": { "input": 3, "output": 15, "cache_read": 3 } } } }"#, + r#""reseller-c": { "models": { "kimi-x": { "canonical_model_id": "moonshotai/kimi-x", + "cost": { "input": 3, "output": 15, "cache_read": 3 } } } }"#, + ]) { + let table = parse_pricing(&raw).unwrap(); + let cost = table.get("kimi-x").unwrap(); + assert_eq!(cost.cache_read, 0.3, "{raw}"); + assert_eq!(cost.cache_write, 3.0, "{raw}"); + } + } + + #[test] + fn flatten_tries_owners_most_named_first_then_by_name() { + let listing = |provider: &str, owner: &str, input: u32| { + format!( + r#""{provider}": {{ "models": {{ "m": {{ "canonical_model_id": "{owner}/m", + "cost": {{ "input": {input}, "output": 1 }} }} }} }}"# + ) + }; + // lab-b is named twice, lab-a once: lab-b's own listing wins. + let named = [ + listing("lab-a", "lab-a", 1), + listing("lab-b", "lab-b", 2), + listing("reseller", "lab-b", 3), + ]; + // lab-a and lab-b tie: the first by name wins. + let tied = [listing("lab-b", "lab-b", 2), listing("lab-a", "lab-a", 1)]; + // The most-named owner lists nothing, so the next one prices it. + let absent = [ + listing("reseller-a", "lab-z", 7), + listing("reseller-b", "lab-z", 7), + listing("lab-a", "lab-a", 1), + ]; + for (blocks, input) in [(&named[..], 2.0), (&tied[..], 1.0), (&absent[..], 1.0)] { + let blocks: Vec<&str> = blocks.iter().map(String::as_str).collect(); + for raw in both_orders(&blocks) { + assert_eq!( + parse_pricing(&raw).unwrap().get("m").unwrap().input, + input, + "{raw}" + ); + } + } + } + + #[test] + fn flatten_prefers_primary_vendors_in_precedence_order() { + for raw in both_orders(&[ + r#""google-vertex": { "models": { "m": { "cost": { "input": 4, "output": 1 } } } }"#, + r#""google": { "models": { "m": { "cost": { "input": 3, "output": 1 } } } }"#, + r#""openai": { "models": { "m": { "cost": { "input": 2, "output": 1 } } } }"#, + r#""xai": { "models": { "m": { "cost": { "input": 5, "output": 1 } } } }"#, + ]) { + assert_eq!( + parse_pricing(&raw).unwrap().get("m").unwrap().input, + 2.0, + "{raw}" + ); + } + } + + #[test] + fn flatten_prices_a_reseller_only_model_at_the_tariff_most_listings_agree_on() { + for raw in both_orders(&[ + r#""alpha": { "models": { "m": { "cost": { "input": 9, "output": 9 } } } }"#, + r#""beta": { "models": { "m": { "cost": { "input": 1, "output": 2 } } } }"#, + r#""gamma": { "models": { "m": { "cost": { "input": 1, "output": 2 } } } }"#, + ]) { + let cost = parse_pricing(&raw).unwrap().get("m").cloned().unwrap(); + assert_eq!((cost.input, cost.output), (1.0, 2.0), "{raw}"); + } + } + + #[test] + fn flatten_never_lets_free_listings_outvote_a_paid_tariff() { + for raw in both_orders(&[ + r#""plan-a": { "models": { "m": { "cost": { "input": 0, "output": 0 } } } }"#, + r#""plan-b": { "models": { "m": { "cost": { "input": 0, "output": 0 } } } }"#, + r#""paid": { "models": { "m": { "cost": { "input": 0, "output": 4 } } } }"#, + ]) { + assert_eq!( + parse_pricing(&raw).unwrap().get("m").unwrap().output, + 4.0, + "{raw}" + ); + } + let only_free = r#"{ + "plan-a": { "models": { "m": { "cost": { "input": 0, "output": 0 } } } } }"#; - // Run repeatedly; each run must surface the second-provider entry. - for _ in 0..10 { - let table = parse_pricing(raw).unwrap(); - let entry = table.get("shared").unwrap(); - assert_eq!(entry.input, 9.0); - assert_eq!(entry.output, 8.0); + assert_eq!( + parse_pricing(only_free).unwrap().get("m").unwrap().output, + 0.0 + ); + } + + #[test] + fn flatten_breaks_reseller_ties_by_provider_id_not_file_order() { + for raw in both_orders(&[ + r#""zeta": { "models": { "m": { "cost": { "input": 9, "output": 8 } } } }"#, + r#""beta": { "models": { "m": { "cost": { "input": 5, "output": 5 } } } }"#, + r#""alpha": { "models": { "m": { "cost": { "input": 1, "output": 2 } } } }"#, + r#""omega": { "models": { "m": { "cost": { "input": 5, "output": 5 } } } }"#, + r#""delta": { "models": { "m": { "cost": { "input": 1, "output": 2 } } } }"#, + ]) { + // {alpha, delta} and {beta, omega} tie at two listings; alpha + // sorts first. + assert_eq!( + parse_pricing(&raw).unwrap().get("m").unwrap().input, + 1.0, + "{raw}" + ); + } + } + + #[test] + fn billing_provider_names_copilot_for_copilot_sessions_only() { + assert_eq!( + billing_provider(SourceKind::CopilotCli), + Some("github-copilot") + ); + for source in [ + SourceKind::ClaudeCode, + SourceKind::Codex, + SourceKind::Opencode, + SourceKind::AnthropicApi, + SourceKind::OpenaiApi, + SourceKind::GeminiApi, + ] { + assert_eq!(billing_provider(source), None, "{source}"); } } + const COPILOT_FIXTURE: &str = r#"{ + "moonshotai": { "models": { + "kimi-x": { "canonical_model_id": "moonshotai/kimi-x", + "cost": { "input": 3, "output": 15, "cache_read": 0.3 } }, + "kimi-only-owner": { "cost": { "input": 2, "output": 4 } } + } }, + "github-copilot": { "models": { + "kimi-x": { "canonical_model_id": "moonshotai/kimi-x", + "cost": { "input": 3, "output": 15, "cache_read": 0.5 } }, + "copilot-unpriced": { "cost": { "input": 1 } } + } }, + "reseller": { "models": { + "kimi-x": { "cost": { "input": 3, "output": 15, "cache_read": 0.9 } } + } } + }"#; + + #[test] + fn rate_prefers_the_billing_providers_tariff_then_the_resolved_one() { + let table = parse_pricing(COPILOT_FIXTURE).unwrap(); + let copilot = Some(GITHUB_COPILOT); + assert_eq!(table.rate("kimi-x", copilot).unwrap().cache_read, 0.5); + assert_eq!(table.rate("kimi-x", None).unwrap().cache_read, 0.3); + assert_eq!( + table.rate("kimi-x", Some("reseller")).unwrap().cache_read, + 0.3 + ); + assert_eq!(table.rate("kimi-only-owner", copilot).unwrap().input, 2.0); + assert!(table.rate("copilot-unpriced", copilot).is_none()); + + let lookup = |source| { + crate::analyze::cost::lookup_model_rate("kimi-x", Some(source), &table) + .unwrap() + .cache_read + }; + assert_eq!(lookup(SourceKind::CopilotCli), 0.5); + assert_eq!(lookup(SourceKind::Opencode), 0.3); + } + + #[test] + fn insert_sets_a_models_tariff_for_every_billing_provider() { + let mut table = parse_pricing(COPILOT_FIXTURE).unwrap(); + let mut custom = table.get("kimi-x").unwrap().clone(); + custom.cache_read = 0.01; + table.insert("kimi-x".into(), custom); + assert_eq!( + table + .rate("kimi-x", Some(GITHUB_COPILOT)) + .unwrap() + .cache_read, + 0.01 + ); + assert_eq!(table.rate("kimi-x", None).unwrap().cache_read, 0.01); + } + + #[test] + fn merge_shadows_resolved_and_billing_provider_tariffs() { + let mut table = parse_pricing(COPILOT_FIXTURE).unwrap(); + table.merge( + parse_pricing( + r#"{ + "user": { "models": { + "kimi-only-owner": { "cost": { "input": 7, "output": 7 } } + } }, + "github-copilot": { "models": { + "kimi-x": { "cost": { "input": 3, "output": 15, "cache_read": 0.02 } } + } } + }"#, + ) + .unwrap(), + ); + let copilot = Some(GITHUB_COPILOT); + assert_eq!(table.rate("kimi-x", copilot).unwrap().cache_read, 0.02); + assert_eq!(table.rate("kimi-only-owner", copilot).unwrap().input, 7.0); + assert_eq!(table.rate("kimi-only-owner", None).unwrap().input, 7.0); + } + + #[test] + fn builtin_snapshot_prices_kimi_k3_at_moonshots_tariff() { + // Resellers listed after Moonshot bill kimi-k3 cache reads at the + // input rate; Moonshot's own tariff prices the bare id. + let table = load_builtin_pricing(); + let moonshot = builtin_listing("moonshotai", "kimi-k3"); + assert_eq!(table.get("kimi-k3"), Some(&moonshot)); + assert!(moonshot.cache_read < moonshot.input); + assert_eq!( + table.rate("kimi-k3", Some(GITHUB_COPILOT)), + Some(&builtin_listing(GITHUB_COPILOT, "kimi-k3")) + ); + } + #[test] fn flatten_prefers_primary_vendor_over_later_reseller_copy() { let raw = r#"{ diff --git a/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs b/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs index b8ac7b77..f523381e 100644 --- a/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs +++ b/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs @@ -376,7 +376,11 @@ fn pick_input_rate(turns: &[&TurnRecord], pricing: &PricingTable) -> f64 { let Some((best_model, _)) = best else { return 0.0; }; - match lookup_model_rate(best_model, pricing) { + let source = turns + .iter() + .find(|t| t.model == best_model) + .map(|t| t.source); + match lookup_model_rate(best_model, source, pricing) { Some(rate) => rate.input / 1_000_000.0, None => 0.0, } diff --git a/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs b/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs index aaae7f15..4aaf0886 100644 --- a/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs +++ b/crates/relayburn-sdk/src/analyze/tool_output_bloat.rs @@ -279,8 +279,8 @@ fn percentile(values: &[u64], p: f64) -> u64 { sorted[idx] } -fn price_carry_cost(tokens: u64, model: &str, pricing: &PricingTable) -> f64 { - let Some(rate) = lookup_model_rate(model, pricing) else { +fn price_carry_cost(tokens: u64, model: &str, source: SourceKind, pricing: &PricingTable) -> f64 { + let Some(rate) = lookup_model_rate(model, Some(source), pricing) else { return 0.0; }; (tokens as f64 / 1_000_000.0) * rate.input @@ -386,7 +386,7 @@ pub fn detect_observed_bloat(opts: &DetectObservedBloatOptions<'_>) -> Vec Date: Fri, 23 Oct 2026 00:07:31 -0700 Subject: [PATCH 2/3] Pin billing-provider pricing in every cost view that takes it Co-Authored-By: Claude Opus 5.5 --- .../src/analyze/context_delta_tests.rs | 13 ++++++++ crates/relayburn-sdk/src/analyze/cost.rs | 14 ++++++++ .../src/analyze/ghost_surface_inputs.rs | 22 +++++++++++++ .../src/analyze/patterns_tests.rs | 33 +++++++++++++++++++ crates/relayburn-sdk/src/analyze/pricing.rs | 6 +--- .../src/analyze/tool_call_patterns.rs | 21 ++++++++++++ 6 files changed, 104 insertions(+), 5 deletions(-) diff --git a/crates/relayburn-sdk/src/analyze/context_delta_tests.rs b/crates/relayburn-sdk/src/analyze/context_delta_tests.rs index f5abe06a..cfbd558a 100644 --- a/crates/relayburn-sdk/src/analyze/context_delta_tests.rs +++ b/crates/relayburn-sdk/src/analyze/context_delta_tests.rs @@ -612,3 +612,16 @@ fn zero_delta_with_compaction_is_not_a_compaction_row() { ); assert!(deltas.is_empty(), "{deltas:?}"); } + +#[test] +fn attributed_cost_charges_growth_at_the_cache_read_tariff() { + let pricing = crate::analyze::pricing::parse_pricing( + r#"{ "acme": { "models": { + "m": { "cost": { "input": 3, "output": 15, "cache_read": 0.25 } } + } } }"#, + ) + .unwrap(); + assert_eq!(attributed_cost(2_000_000, "m", &pricing), 0.5); + assert_eq!(attributed_cost(-5, "m", &pricing), 0.0); + assert_eq!(attributed_cost(2_000_000, "unpriced", &pricing), 0.0); +} diff --git a/crates/relayburn-sdk/src/analyze/cost.rs b/crates/relayburn-sdk/src/analyze/cost.rs index 0b2d7345..206c1015 100644 --- a/crates/relayburn-sdk/src/analyze/cost.rs +++ b/crates/relayburn-sdk/src/analyze/cost.rs @@ -766,6 +766,20 @@ mod tests { assert_eq!(rate.output, 2.0); } + #[test] + fn lookup_model_rate_strips_a_provider_prefix_at_the_billing_providers_tariff() { + let p = crate::analyze::pricing::parse_pricing( + r#"{ + "acme": { "models": { "foo": { "cost": { "input": 1, "output": 2 } } } }, + "github-copilot": { "models": { "foo": { "cost": { "input": 3, "output": 4 } } } } + }"#, + ) + .unwrap(); + let rate = |source| lookup_model_rate("router/foo", source, &p).unwrap().input; + assert_eq!(rate(None), 1.0); + assert_eq!(rate(Some(SourceKind::CopilotCli)), 3.0); + } + #[test] fn lookup_model_rate_maps_codex_auto_review_to_gpt_5_2_codex() { let mut p = PricingTable::new(); diff --git a/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs b/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs index 0bd2cec4..28e0eb31 100644 --- a/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs +++ b/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs @@ -295,4 +295,26 @@ mod tests { let inputs = build_ghost_surface_inputs(&[], &pricing, Some(HashMap::new())); assert!(inputs.user_turn_text_by_session.is_none()); } + + #[test] + fn tied_models_take_the_first_seen_models_billing_provider_rate() { + let pricing = crate::analyze::pricing::parse_pricing( + r#"{ + "acme": { "models": { + "first": { "cost": { "input": 1, "output": 1, "cache_read": 0.1 } }, + "second": { "cost": { "input": 1, "output": 1, "cache_read": 0.9 } } + } }, + "github-copilot": { "models": { + "first": { "cost": { "input": 1, "output": 1, "cache_read": 0.4 } } + } } + }"#, + ) + .unwrap(); + let turns = vec![ + make_turn(SourceKind::CopilotCli, "s1", "first", vec![], None), + make_turn(SourceKind::CopilotCli, "s1", "second", vec![], None), + ]; + let rate = pick_representative_cache_read_rate(&turns, &pricing); + assert!((rate - 0.4 / 1_000_000.0).abs() < 1e-15, "{rate}"); + } } diff --git a/crates/relayburn-sdk/src/analyze/patterns_tests.rs b/crates/relayburn-sdk/src/analyze/patterns_tests.rs index 69dc9d84..a4f5cdfa 100644 --- a/crates/relayburn-sdk/src/analyze/patterns_tests.rs +++ b/crates/relayburn-sdk/src/analyze/patterns_tests.rs @@ -584,6 +584,39 @@ mod compaction_losses { assert_eq!(c.preceding_message_id.as_deref(), Some("msg_c_1")); assert!(c.cache_lost_cost > 0.0); } + + #[test] + fn prices_a_copilot_compaction_at_copilots_cache_read_tariff() { + let pricing = crate::analyze::pricing::parse_pricing( + r#"{ + "acme": { "models": { "m": { "cost": { "input": 1, "output": 1, "cache_read": 0.1 } } } }, + "github-copilot": { "models": { "m": { "cost": { "input": 1, "output": 1, "cache_read": 0.5 } } } } + }"#, + ) + .unwrap(); + let mut preceding = turn("s", "m1", 0); + preceding.source = SourceKind::CopilotCli; + preceding.model = "m".into(); + let compaction = CompactionEvent { + v: 1, + source: SourceKind::CopilotCli, + session_id: "s".into(), + ts: "2026-04-20T00:00:01.000Z".into(), + preceding_message_id: Some("m1".into()), + tokens_before_compact: Some(1_000_000), + }; + let result = detect_patterns( + &[preceding], + &DetectPatternsOptions { + pricing: &pricing, + compactions: Some(std::slice::from_ref(&compaction)), + user_turns_by_session: None, + content_by_session: None, + tool_result_events: None, + }, + ); + assert_eq!(result.compactions[0].cache_lost_cost, 0.5); + } } // --------------------------------------------------------------------------- diff --git a/crates/relayburn-sdk/src/analyze/pricing.rs b/crates/relayburn-sdk/src/analyze/pricing.rs index 5c5fea10..cde7a082 100644 --- a/crates/relayburn-sdk/src/analyze/pricing.rs +++ b/crates/relayburn-sdk/src/analyze/pricing.rs @@ -101,10 +101,6 @@ impl PricingTable { self.models.contains_key(model) } - pub fn iter(&self) -> impl Iterator { - self.models.iter() - } - pub fn values(&self) -> impl Iterator { self.models.values() } @@ -292,7 +288,7 @@ fn load_from_file(path: &Path) -> io::Result { parse_pricing(&raw).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) } -fn parse_pricing(raw: &str) -> serde_json::Result { +pub(crate) fn parse_pricing(raw: &str) -> serde_json::Result { parse_pricing_with_protected_models(raw, &HashSet::new()) } diff --git a/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs b/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs index f523381e..2ada470c 100644 --- a/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs +++ b/crates/relayburn-sdk/src/analyze/tool_call_patterns.rs @@ -1013,4 +1013,25 @@ mod tests { let s = serde_json::to_string(&ToolCallPatternCategory::BashGhPr).unwrap(); assert_eq!(s, "\"bash-gh-pr\""); } + + #[test] + fn input_rate_uses_the_dominant_models_billing_provider_tariff() { + let pricing = crate::analyze::pricing::parse_pricing( + r#"{ + "acme": { "models": { + "m": { "cost": { "input": 1, "output": 1 } }, + "other": { "cost": { "input": 9, "output": 1 } } + } }, + "github-copilot": { "models": { "m": { "cost": { "input": 4, "output": 1 } } } } + }"#, + ) + .unwrap(); + let mut other = turn("s", "a", 0, SourceKind::ClaudeCode, vec![]); + other.model = "other".into(); + let mut m = turn("s", "b", 1, SourceKind::CopilotCli, vec![]); + m.model = "m".into(); + let turns = [other.clone(), m.clone(), m]; + let refs: Vec<&TurnRecord> = turns.iter().collect(); + assert_eq!(pick_input_rate(&refs, &pricing), 4.0 / 1_000_000.0); + } } From 8707f2aeaed1465af66b5bb852fd3f723d6d1ec8 Mon Sep 17 00:00:00 2001 From: Will Washburn Date: Fri, 23 Oct 2026 00:10:23 -0700 Subject: [PATCH 3/3] Never price a model at a free-tier listing while a paid tariff exists A $0/$0 listing is a free tier or plan, so owner, primary-vendor, consensus and Copilot billing tariffs all skip it when any listing is paid. The ghost-surface representative rate counts turns per model and billing provider, so one Copilot turn no longer prices a mixed slice. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- .../src/analyze/ghost_surface_inputs.rs | 41 ++++++++---- crates/relayburn-sdk/src/analyze/pricing.rs | 66 +++++++++++++++---- 3 files changed, 85 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ac64506..e2e900cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ Cross-package release notes for relayburn. Package changelogs contain package-le - A Claude assistant turn still being written (no `stop_reason` yet) is billed once it settles, not early. - A Claude subagent turn that follows a tool result inside its sidechain keeps the subagent's agent id instead of starting a new subagent. - Pricing bills Claude 1-hour cache writes at 2x the input tariff on every host (Anthropic, Vertex AI, Bedrock; 5-minute writes stay at the `cache_write` tariff), so `summary`, `hotspots`, `analyze`, and every cost report stop undercounting sessions that use 1-hour prompt caching. `ModelCost` / `ModelCostTier` gain `cache_write_1h`, and a ledger-home `models.dev.json` entry can set `cache_write_1h` explicitly; non-Claude models keep a single cache-write tariff. -- Pricing resolves a model id that several providers list without consulting file order: the owning lab's own tariff (named by models.dev `canonical_model_id`, e.g. Moonshot for `kimi-*`), else a first-party vendor's, else the tariff most paid reseller listings agree on. `kimi-k3` stops billing cache reads at the input rate, and 409 of 3,796 bundled model ids move to their owner's or the consensus tariff. Copilot CLI turns price at GitHub Copilot's own tariff wherever it lists the model. +- Pricing resolves a model id that several providers list without consulting file order: the owning lab's own tariff (named by models.dev `canonical_model_id`, e.g. Moonshot for `kimi-*`), else a first-party vendor's, else the tariff most reseller listings agree on; a free-tier ($0/$0) listing prices a model only when no listing is paid. `kimi-k3` stops billing cache reads at the input rate, and 410 of 3,796 bundled model ids move to their owner's or the consensus tariff. Copilot CLI turns price at GitHub Copilot's own tariff wherever it lists the model. - Turns whose usage was measured as zero price at $0 regardless of model, so Claude Code's `` messages no longer list as unpriced in `summary` or strip the dollar estimate from every hotspots finding in their session; unmeasured usage stays unpriced. `burn compare` shows cost per turn only for fully priced cells. ### Analysis and quality diff --git a/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs b/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs index 28e0eb31..9550712d 100644 --- a/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs +++ b/crates/relayburn-sdk/src/analyze/ghost_surface_inputs.rs @@ -65,23 +65,24 @@ pub fn build_session_count_by_source(turns: &[TurnRecord]) -> HashMap f64 { - let mut counts: HashMap<&str, u64> = HashMap::new(); - let mut order: Vec<(&str, SourceKind)> = Vec::new(); + // A model's rate depends on who bills it, so turns count per (model, + // billing provider) pair. + let mut counts: HashMap<(&str, Option<&str>), u64> = HashMap::new(); + let mut order: Vec<(&str, Option<&str>)> = Vec::new(); for t in turns { - let m = t.model.as_str(); - if !counts.contains_key(m) { - order.push((m, t.source)); + let key = (t.model.as_str(), billing_provider(t.source)); + if !counts.contains_key(&key) { + order.push(key); } - *counts.entry(m).or_insert(0) += 1; + *counts.entry(key).or_insert(0) += 1; } let mut best_rate: Option = None; let mut best_count: i64 = -1; - // Iterate in first-seen order so ties go to the earliest-seen model + // Iterate in first-seen order so ties go to the earliest-seen pair // (matches the TS `Map` iteration order). - for (m, source) in order { - let c = counts[m] as i64; - let rate = pricing.rate(m, billing_provider(source)); - if let Some(rate) = rate.filter(|_| c > best_count) { + for key @ (model, billed_by) in order { + let c = counts[&key] as i64; + if let Some(rate) = pricing.rate(model, billed_by).filter(|_| c > best_count) { best_rate = Some(rate.cache_read); best_count = c; } @@ -317,4 +318,22 @@ mod tests { let rate = pick_representative_cache_read_rate(&turns, &pricing); assert!((rate - 0.4 / 1_000_000.0).abs() < 1e-15, "{rate}"); } + + #[test] + fn the_most_used_billing_provider_sets_a_shared_models_rate() { + let pricing = crate::analyze::pricing::parse_pricing( + r#"{ + "acme": { "models": { "m": { "cost": { "input": 1, "output": 1, "cache_read": 0.3 } } } }, + "github-copilot": { "models": { "m": { "cost": { "input": 1, "output": 1, "cache_read": 0.5 } } } } + }"#, + ) + .unwrap(); + let turns = vec![ + make_turn(SourceKind::CopilotCli, "s1", "m", vec![], None), + make_turn(SourceKind::Opencode, "s2", "m", vec![], None), + make_turn(SourceKind::Opencode, "s2", "m", vec![], None), + ]; + let rate = pick_representative_cache_read_rate(&turns, &pricing); + assert!((rate - 0.3 / 1_000_000.0).abs() < 1e-15, "{rate}"); + } } diff --git a/crates/relayburn-sdk/src/analyze/pricing.rs b/crates/relayburn-sdk/src/analyze/pricing.rs index cde7a082..7f78a47d 100644 --- a/crates/relayburn-sdk/src/analyze/pricing.rs +++ b/crates/relayburn-sdk/src/analyze/pricing.rs @@ -329,7 +329,7 @@ fn flatten(root: &ModelsDevRoot, protected_models: &HashSet) -> PricingT let Some(cost) = model_cost(id, model) else { continue; }; - if BILLING_PROVIDERS.contains(&provider_id.as_str()) { + if BILLING_PROVIDERS.contains(&provider_id.as_str()) && !is_free(&cost) { table .billed .entry(provider_id.clone()) @@ -359,7 +359,9 @@ fn ranked_owners(votes: HashMap<&str, usize>) -> Vec<&str> { owners.into_iter().map(|(owner, _)| owner).collect() } -/// Choose a model id's tariff from its priced `listings`: +/// Choose a model id's tariff from its priced `listings`. A listing free on +/// both input and output is a free tier or plan, not a token tariff, so it +/// takes part only when every listing is free. Of the rest: /// /// 1. the owner's own listing, trying `owners` in rank order; /// 2. else the listing of the first [`PRIMARY_PRICING_PROVIDERS`] vendor @@ -367,17 +369,20 @@ fn ranked_owners(votes: HashMap<&str, usize>) -> Vec<&str> { /// 3. else nothing when the id was ever first-party (`protected`), so a /// retired first-party model is unpriced rather than inheriting a /// reseller tariff; -/// 4. else the tariff the most paid listings agree on, ties going to the -/// tariff whose providers include the alphabetically first provider id. -/// A listing free on both input and output is a free tier or plan, not a -/// token tariff, so it prices the model only when every listing is free. +/// 4. else the tariff the most listings agree on, ties going to the tariff +/// whose providers include the alphabetically first provider id. fn resolve_tariff<'a>( listings: &'a [Listing<'_>], owners: &[&str], protected: bool, ) -> Option<&'a ModelCost> { + let any_paid = listings.iter().any(|listing| !is_free(&listing.cost)); + let tariffs: Vec<&Listing<'_>> = listings + .iter() + .filter(|listing| !any_paid || !is_free(&listing.cost)) + .collect(); let listed_by = |provider: &str| { - listings + tariffs .iter() .find(|listing| listing.provider == provider) .map(|listing| &listing.cost) @@ -394,15 +399,13 @@ fn resolve_tariff<'a>( if protected { return None; } - consensus_tariff(listings) + consensus_tariff(&tariffs) } -fn consensus_tariff<'a>(listings: &'a [Listing<'_>]) -> Option<&'a ModelCost> { - let free = |listing: &&Listing<'_>| listing.cost.input == 0.0 && listing.cost.output == 0.0; - let all_free = listings.iter().all(|listing| free(&listing)); +fn consensus_tariff<'a>(listings: &[&'a Listing<'_>]) -> Option<&'a ModelCost> { // (tariff, listings agreeing, alphabetically first provider among them) let mut groups: Vec<(&ModelCost, usize, &str)> = Vec::new(); - for listing in listings.iter().filter(|listing| all_free || !free(listing)) { + for listing in listings { match groups .iter_mut() .find(|(cost, _, _)| **cost == listing.cost) @@ -422,6 +425,11 @@ fn consensus_tariff<'a>(listings: &'a [Listing<'_>]) -> Option<&'a ModelCost> { .map(|(cost, _, _)| cost) } +/// Free on both input and output: a free tier or plan, not a token tariff. +fn is_free(cost: &ModelCost) -> bool { + cost.input == 0.0 && cost.output == 0.0 +} + /// A listing's tariff, or `None` when it lacks `input` or `output`. fn model_cost(id: &str, model: &ModelsDevModel) -> Option { let cost = model.cost.as_ref()?; @@ -896,6 +904,40 @@ mod tests { ); } + #[test] + fn flatten_never_takes_a_free_owner_or_primary_listing_over_a_paid_one() { + for (free_block, provider) in [ + ( + r#""lab": { "models": { "m": { "canonical_model_id": "lab/m", + "cost": { "input": 0, "output": 0 } } } }"#, + "owner", + ), + ( + r#""openai": { "models": { "m": { "cost": { "input": 0, "output": 0 } } } }"#, + "primary", + ), + ] { + let paid = r#""reseller": { "models": { "m": { "canonical_model_id": "lab/m", + "cost": { "input": 0.1, "output": 0.2 } } } }"#; + for raw in both_orders(&[free_block, paid]) { + let cost = parse_pricing(&raw).unwrap().get("m").cloned().unwrap(); + assert_eq!((cost.input, cost.output), (0.1, 0.2), "{provider}: {raw}"); + } + } + } + + #[test] + fn a_free_billing_provider_listing_never_shadows_the_resolved_tariff() { + let table = parse_pricing( + r#"{ + "acme": { "models": { "m": { "cost": { "input": 2, "output": 8 } } } }, + "github-copilot": { "models": { "m": { "cost": { "input": 0, "output": 0 } } } } + }"#, + ) + .unwrap(); + assert_eq!(table.rate("m", Some(GITHUB_COPILOT)).unwrap().input, 2.0); + } + #[test] fn flatten_breaks_reseller_ties_by_provider_id_not_file_order() { for raw in both_orders(&[