diff --git a/CHANGELOG.md b/CHANGELOG.md index dcd7393d3..682702268 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,20 +8,6 @@ Newest first. `Unreleased` is what is on `main` and not yet tagged. ## Unreleased -### The Helm chart pins by digest, mounts a tenant package, and takes a Composio key - -Three things a deployment on Kubernetes did by hand now have values of their own: - -- `image.digest` deploys exactly the image a release's `container-images.json` names, as - `repository:tag@digest`. It refuses anything that is not `sha256:` and 64 hex characters. -- `config.extraVolumes` and `config.extraVolumeMounts` mount volumes on the server and on the routines - and computer-culler CronJobs, the pods that load the configuration. A tenant package from a - ConfigMap no longer needs a custom image. -- `secrets.composioApiKey`, or `composio-api-key` named in `externalSecrets.data`, reaches the server - as `COMPOSIO_API_KEY`. Only the server receives it, not the CronJobs. - -With none of them set, every target renders exactly as before. - **Before upgrading.** Four things change for an existing deployment: - A hosted computer's browser is now headed by default. It uses a little more memory than headless did; set `COMPUTER_BROWSER_MODE=headless` (Helm: `computers.browserMode: headless`) to keep the old @@ -43,6 +29,29 @@ With none of them set, every target renders exactly as before. `OPENBOT_MEMORY_SOURCE_DEFAULTS=false` and `OPENBOT_PROACTIVE_DEFAULTS=false` (Helm: `config.memory.*`) prevent it. +### The server says at boot when its key cannot call the configured model + +A key without access to the tenant package's `default_model` passed every start-up check and failed at +the first message, as "The model … does not exist or you do not have access to it", in front of +whoever sent it. The server now asks the provider once at start-up and logs `model-unavailable` with +the model and the fix, or `model-credential-rejected` for a refused key. It asks only OpenAI and +Anthropic themselves, not an endpoint set by `OPENAI_BASE_URL` or `ANTHROPIC_BASE_URL`, says nothing +while no key is configured, and never delays start-up. + +### The Helm chart pins by digest, mounts a tenant package, and takes a Composio key + +Three things a deployment on Kubernetes did by hand now have values of their own: + +- `image.digest` deploys exactly the image a release's `container-images.json` names, as + `repository:tag@digest`. It refuses anything that is not `sha256:` and 64 hex characters. +- `config.extraVolumes` and `config.extraVolumeMounts` mount volumes on the server and on the routines + and computer-culler CronJobs, the pods that load the configuration. A tenant package from a + ConfigMap no longer needs a custom image. +- `secrets.composioApiKey`, or `composio-api-key` named in `externalSecrets.data`, reaches the server + as `COMPOSIO_API_KEY`. Only the server receives it, not the CronJobs. + +With none of them set, every target renders exactly as before. + ### Fewer "Verify you are human" pages Sites behind Cloudflare challenged a Bot's browser on sight, and opening one could restart the diff --git a/docs/configuration.md b/docs/configuration.md index fd3c2ff1b..cc5d93db1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1158,6 +1158,8 @@ model: `provider` must be `openai` or `anthropic`. `credential_secret_ref` is a reference to a stored credential, not a credential value. `default_model` is passed through as written, so an OpenAI-compatible endpoint reached through `OPENAI_BASE_URL` takes the name that endpoint publishes. +At start-up the server asks OpenAI or Anthropic, once, whether the configured key can call `default_model`. A key that cannot (a project key whose model allowlist leaves it out, say) would otherwise fail at the first message; instead the log says `model-unavailable`, naming the model and the fix. A rejected key logs `model-credential-rejected`. Nothing is asked with no key configured yet, or of an endpoint set by `OPENAI_BASE_URL` or `ANTHROPIC_BASE_URL`, and start-up never waits for the answer. + ### `knowledge.yaml` ```yaml diff --git a/server/src/index.ts b/server/src/index.ts index 033d2c628..3088b44eb 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -203,6 +203,7 @@ import { createMemoryDefaults } from "./memory/defaults"; import { createMemoryIngestion } from "./memory/ingestion"; import { createIntelligenceMemory } from "./memory/intelligence"; import { createMemoryStore } from "./memory/store"; +import { reportModelAvailability } from "./model-availability-check"; import { memoryTools } from "./memory/tools"; import { createSignInNotifier, createSignInResumer } from "./passwords/resume"; import { createSignInService } from "./passwords/service"; @@ -1126,6 +1127,21 @@ const intentRouter = createIntentRouter({ }), }); +// Said once at boot: whether the key can call the model the Bots run on. A key without access to it +// otherwise passes every start-up check and fails at the first message, in front of whoever sent it. +void reportModelAvailability({ + provider: runtimeModel.provider, + model: runtimeModel.defaultModel, + apiKey: () => + resolveModelApiKey({ + encryptionKey: config.keyEncryptionKey, + reader: credentialStore, + provider: runtimeModel.provider, + keyId: tenantPackage.model.credentialSecretRef, + environment: process.env, + }), +}); + /** * Pass one of tool selection: which skills a message needs, on the deployment's own model. * diff --git a/server/src/model-availability-check.ts b/server/src/model-availability-check.ts new file mode 100644 index 000000000..3aed32a67 --- /dev/null +++ b/server/src/model-availability-check.ts @@ -0,0 +1,139 @@ +/** + * Whether the model this deployment's Bots run on is one its key can call, said once at boot. + * + * The tenant package names the model (`model.yaml`'s `default_model`), and the provider decides + * whether a key may use it. A project key without access to that model passes every check OpenBot + * makes at start-up and fails at the first message, as "The model `…` does not exist or you do not + * have access to it", in front of whoever sent it. One read of the provider's own model endpoint at + * boot moves that to the log, where the person who can fix it is looking. + * + * Only the providers' own APIs are asked. An OpenAI-compatible endpoint (`OPENAI_BASE_URL`) often + * serves no single-model lookup, so a 404 there would say nothing about the model; those are left + * alone. No key configured yet is also left alone: an administrator may still be about to add one. + * Never blocks start-up and never throws. + */ +export type ModelCheckOutcome = + | { kind: "available" } + | { kind: "skipped" } + | { + kind: "warning"; + line: { + type: + | "model-unavailable" + | "model-credential-rejected" + | "model-check-unreadable"; + provider: "openai" | "anthropic"; + model: string; + status?: number; + reason?: string; + fix: string; + }; + }; + +type Provider = "openai" | "anthropic"; + +const ENDPOINTS: Record string> = { + openai: (model) => + `https://api.openai.com/v1/models/${encodeURIComponent(model)}`, + anthropic: (model) => + `https://api.anthropic.com/v1/models/${encodeURIComponent(model)}`, +}; + +/** What a provider's answer to "may this key use this model" means for the deployment. */ +export function modelCheckOutcome(input: { + provider: Provider; + model: string; + status: number; +}): ModelCheckOutcome { + const { provider, model, status } = input; + const keyName = + provider === "anthropic" ? "ANTHROPIC_API_KEY" : "OPENAI_API_KEY"; + if (status >= 200 && status < 300) return { kind: "available" }; + if (status === 404) { + return { + kind: "warning", + line: { + type: "model-unavailable", + provider, + model, + status, + fix: + `The ${provider} key this deployment uses cannot call ${model}, so every Bot turn on it will fail. ` + + `Give the key access to ${model} (a project key may have a model allowlist), use a key that has it, ` + + `or change default_model in the tenant package's model.yaml.`, + }, + }; + } + if (status === 401 || status === 403) { + return { + kind: "warning", + line: { + type: "model-credential-rejected", + provider, + model, + status, + fix: `${provider} refused the model key (${keyName} or the credential stored in OpenBot), so every Bot turn will fail until it is replaced.`, + }, + }; + } + return { + kind: "warning", + line: { + type: "model-check-unreadable", + provider, + model, + status, + fix: `${provider} did not say whether ${model} is available (HTTP ${status}). This is often temporary.`, + }, + }; +} + +/** Ask once and log what was found. Resolves either way and never throws. */ +export async function reportModelAvailability(options: { + provider: Provider; + model: string; + apiKey: () => Promise; + environment?: Record; + fetch?: typeof fetch; + log?: (message: string) => void; + timeoutMs?: number; +}): Promise { + const environment = options.environment ?? process.env; + const log = options.log ?? ((message: string) => console.warn(message)); + const request = options.fetch ?? fetch; + const { provider, model } = options; + + const customBase = + provider === "anthropic" + ? environment.ANTHROPIC_BASE_URL + : environment.OPENAI_BASE_URL; + if (customBase?.trim()) return { kind: "skipped" }; + + let outcome: ModelCheckOutcome; + try { + const key = await options.apiKey(); + if (!key) return { kind: "skipped" }; + const headers: Record = + provider === "anthropic" + ? { "x-api-key": key, "anthropic-version": "2023-06-01" } + : { authorization: `Bearer ${key}` }; + const response = await request(ENDPOINTS[provider](model), { + headers, + signal: AbortSignal.timeout(options.timeoutMs ?? 5_000), + }); + outcome = modelCheckOutcome({ provider, model, status: response.status }); + } catch (error) { + outcome = { + kind: "warning", + line: { + type: "model-check-unreadable", + provider, + model, + reason: error instanceof Error ? error.message : String(error), + fix: `Could not ask ${provider} whether ${model} is available. This is often temporary.`, + }, + }; + } + if (outcome.kind === "warning") log(JSON.stringify(outcome.line)); + return outcome; +} diff --git a/server/tests/model-availability-check.test.ts b/server/tests/model-availability-check.test.ts new file mode 100644 index 000000000..b97e6214f --- /dev/null +++ b/server/tests/model-availability-check.test.ts @@ -0,0 +1,172 @@ +import { describe, expect, test } from "bun:test"; +import { + modelCheckOutcome, + reportModelAvailability, +} from "../src/model-availability-check"; + +const KEY = "sk-proj-not-a-real-key-0123456789"; + +function fakeFetch(status: number) { + const calls: { url: string; headers: Record }[] = []; + const fetcher = (async (url: string | URL | Request, init?: RequestInit) => { + calls.push({ + url: String(url), + headers: (init?.headers ?? {}) as Record, + }); + return new Response("{}", { status }); + }) as typeof fetch; + return { fetcher, calls }; +} + +describe("what the provider's answer means", () => { + test("2xx is available", () => { + expect( + modelCheckOutcome({ provider: "openai", model: "gpt-x", status: 200 }), + ).toEqual({ kind: "available" }); + }); + + test("404 says the key cannot call the model, and how to fix it", () => { + // The state a project key with a model allowlist reaches: every Bot turn fails with "does not + // exist or you do not have access to it", and nothing said so before the first message. + const outcome = modelCheckOutcome({ + provider: "openai", + model: "gpt-5.6-terra", + status: 404, + }); + expect(outcome.kind).toBe("warning"); + if (outcome.kind !== "warning") return; + expect(outcome.line.type).toBe("model-unavailable"); + expect(outcome.line.fix).toContain("gpt-5.6-terra"); + expect(outcome.line.fix).toContain("model.yaml"); + }); + + test.each([401, 403])("%i says the key itself was refused", (status) => { + const outcome = modelCheckOutcome({ + provider: "anthropic", + model: "claude-x", + status, + }); + expect(outcome.kind === "warning" && outcome.line.type).toBe( + "model-credential-rejected", + ); + expect(outcome.kind === "warning" && outcome.line.fix).toContain( + "ANTHROPIC_API_KEY", + ); + }); + + test("anything else is reported as unreadable, not as a missing model", () => { + const outcome = modelCheckOutcome({ + provider: "openai", + model: "gpt-x", + status: 503, + }); + expect(outcome.kind === "warning" && outcome.line.type).toBe( + "model-check-unreadable", + ); + }); +}); + +describe("asking at boot", () => { + test("asks OpenAI for the one model, with the key as a bearer token, and logs nothing when it is there", async () => { + const { fetcher, calls } = fakeFetch(200); + const lines: string[] = []; + const outcome = await reportModelAvailability({ + provider: "openai", + model: "gpt-5.6-terra", + apiKey: async () => KEY, + environment: {}, + fetch: fetcher, + log: (line) => lines.push(line), + }); + expect(outcome).toEqual({ kind: "available" }); + expect(lines).toEqual([]); + expect(calls[0]?.url).toBe( + "https://api.openai.com/v1/models/gpt-5.6-terra", + ); + expect(calls[0]?.headers.authorization).toBe(`Bearer ${KEY}`); + }); + + test("asks Anthropic with its own headers", async () => { + const { fetcher, calls } = fakeFetch(200); + await reportModelAvailability({ + provider: "anthropic", + model: "claude-sonnet-4-5", + apiKey: async () => KEY, + environment: {}, + fetch: fetcher, + log: () => {}, + }); + expect(calls[0]?.url).toBe( + "https://api.anthropic.com/v1/models/claude-sonnet-4-5", + ); + expect(calls[0]?.headers["x-api-key"]).toBe(KEY); + expect(calls[0]?.headers["anthropic-version"]).toBeDefined(); + }); + + test("logs one line for a model the key cannot call, without the key in it", async () => { + const { fetcher } = fakeFetch(404); + const lines: string[] = []; + await reportModelAvailability({ + provider: "openai", + model: "gpt-5.6-terra", + apiKey: async () => KEY, + environment: {}, + fetch: fetcher, + log: (line) => lines.push(line), + }); + expect(lines).toHaveLength(1); + expect(JSON.parse(lines[0] ?? "{}")).toMatchObject({ + type: "model-unavailable", + model: "gpt-5.6-terra", + }); + expect(lines[0]).not.toContain(KEY); + }); + + test("leaves an OpenAI-compatible endpoint alone, where a 404 would mean nothing", async () => { + const { fetcher, calls } = fakeFetch(404); + const outcome = await reportModelAvailability({ + provider: "openai", + model: "qwen3", + apiKey: async () => KEY, + environment: { OPENAI_BASE_URL: "http://localhost:11434/v1" }, + fetch: fetcher, + log: () => {}, + }); + expect(outcome).toEqual({ kind: "skipped" }); + expect(calls).toHaveLength(0); + }); + + test("says nothing while no key is configured yet", async () => { + const { fetcher, calls } = fakeFetch(200); + const lines: string[] = []; + const outcome = await reportModelAvailability({ + provider: "openai", + model: "gpt-x", + apiKey: async () => null, + environment: {}, + fetch: fetcher, + log: (line) => lines.push(line), + }); + expect(outcome).toEqual({ kind: "skipped" }); + expect(calls).toHaveLength(0); + expect(lines).toEqual([]); + }); + + test("a provider it cannot reach is reported and does not throw", async () => { + const lines: string[] = []; + const outcome = await reportModelAvailability({ + provider: "openai", + model: "gpt-x", + apiKey: async () => KEY, + environment: {}, + fetch: (async () => { + throw new Error("getaddrinfo ENOTFOUND api.openai.com"); + }) as unknown as typeof fetch, + log: (line) => lines.push(line), + }); + expect(outcome.kind === "warning" && outcome.line.type).toBe( + "model-check-unreadable", + ); + expect(JSON.parse(lines[0] ?? "{}").reason).toContain("ENOTFOUND"); + }); +});