diff --git a/README.md b/README.md index 070d59327..8601923f3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The OpenAI Agents API, open source and self-hosted. Run Codex, Claude Code or Mi - **Same API as OpenAI.** Set `OPENAI_BASE_URL` and the [official OpenAI SDK](https://developers.openai.com/api/docs/libraries), or plain HTTP, works unchanged. - **Your agent, your model.** Each Session runs Codex, Claude Code or MiniMax Code natively, as it runs on your machine, with the model provider you configure. See [Harness capabilities](https://openagentcore.dev/contracts/agents-api/harness-capabilities). -- **Compute that sleeps, workspaces that stay.** On microsandbox, which keeps every workspace on independent storage, idle sandboxes release compute and resume on a compatible node with their files intact. E2B, Docker and your own Linux, macOS or Windows machine also run Sessions. See [Sandbox Provider protocol](https://openagentcore.dev/docs/sandbox-provider). +- **Compute that sleeps, workspaces that stay.** On microsandbox, which keeps every workspace on independent storage, idle sandboxes release compute and resume on a compatible node with their files intact. E2B pauses idle sandboxes in place. Docker and your own Linux, macOS or Windows machine also run Sessions. See [Sandbox Provider protocol](https://openagentcore.dev/docs/sandbox-provider). - **Replaceable parts.** Sandbox Providers, workspace filesystems, Harnesses and model providers plug in through [defined protocols](AGENTS.md#protocols-at-every-boundary). ![Deployment overview](docs/assets/console-overview-en.webp) diff --git a/README.zh-CN.md b/README.zh-CN.md index e4cb85dbd..a92ff4679 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -17,7 +17,7 @@ - **与 OpenAI 相同的 API。** 设置 `OPENAI_BASE_URL` 后,[官方 OpenAI SDK](https://developers.openai.com/api/docs/libraries)或直接的 HTTP 调用无需改动即可使用。 - **你的 Agent,你的模型。** 每个 Session 原生运行 Codex、Claude Code 或 MiniMax Code,使用你配置的模型提供商,与在本机运行一致。详见 [Harness 能力](https://openagentcore.dev/zh/contracts/agents-api/harness-capabilities)。 -- **计算会休眠,工作区会保留。** 在 microsandbox 上(每个工作区都保存在独立存储中),空闲沙箱释放计算资源,并在兼容节点上带着文件恢复。E2B、Docker 以及你自己的 Linux、macOS 或 Windows 机器同样可以运行 Session。详见[沙箱提供商协议](https://openagentcore.dev/zh/docs/sandbox-provider)。 +- **计算会休眠,工作区会保留。** 在 microsandbox 上(每个工作区都保存在独立存储中),空闲沙箱释放计算资源,并在兼容节点上带着文件恢复。E2B 会就地暂停空闲沙箱。Docker 以及你自己的 Linux、macOS 或 Windows 机器同样可以运行 Session。详见[沙箱提供商协议](https://openagentcore.dev/zh/docs/sandbox-provider)。 - **部件可替换。** 沙箱提供商、工作区文件系统、Harness 和模型提供商都通过[既定协议](AGENTS.md#protocols-at-every-boundary)接入。 ![部署概览](docs/assets/console-overview-zh.webp) diff --git a/apps/daemon/internal/cli/connect.go b/apps/daemon/internal/cli/connect.go index be8a74339..9240239fd 100644 --- a/apps/daemon/internal/cli/connect.go +++ b/apps/daemon/internal/cli/connect.go @@ -69,11 +69,12 @@ func runConnect(ctx *runContext, args []string) error { return fmt.Errorf("connect: %w", err) } var bootstrapped *auth.Profile + var suspendEnabled bool if *bootstrapFile != "" { if *remote != "" || *environment != "" || *credentialFile != "" || *workspace != "" || fs.NArg() != 0 { return errors.New("connect: bootstrap input cannot be combined with enrollment options") } - bootstrapped, err = bootstrapProfile(*bootstrapFile) + bootstrapped, suspendEnabled, err = bootstrapProfile(*bootstrapFile) if err != nil { return err } @@ -118,7 +119,7 @@ func runConnect(ctx *runContext, args []string) error { } } - return mainLoop(ctx, *profile, prof, agentCLIs) + return mainLoop(ctx, *profile, prof, agentCLIs, suspendEnabled) } // spawnBackground forks the daemon into the background. Parent @@ -189,11 +190,11 @@ func spawnBackground(ctx context.Context, rc *runContext, profile string, argv [ // background process. SIGINT / SIGTERM cancels the root context, which // unblocks the read pump and any in-flight Send so the daemon exits // without orphaning agent subprocesses. -func mainLoop(rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery) error { - return mainLoopRemote(context.Background(), rc, profile, prof, agentCLIs, "") +func mainLoop(rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, suspend bool) error { + return mainLoopRemote(context.Background(), rc, profile, prof, agentCLIs, "", suspend) } -func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, remote string) error { +func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, remote string, suspend bool) error { // Route through obs/log so daemon log lines pick up the same // trace_id / span_id auto-injection as the server side — when the // daemon adopts an envelope's trace, every log call under that ctx @@ -229,7 +230,7 @@ func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof registry := agent.NewRegistry() registerAgentKinds(registry, agentCLIs) - control, err := newSuspendControl() + control, err := newSuspendControl(suspend) if err != nil { return err } diff --git a/apps/daemon/internal/cli/connect_bootstrap.go b/apps/daemon/internal/cli/connect_bootstrap.go index 718b9f9a2..3be0f82f1 100644 --- a/apps/daemon/internal/cli/connect_bootstrap.go +++ b/apps/daemon/internal/cli/connect_bootstrap.go @@ -10,17 +10,17 @@ import ( // The launch file is the sole credential source for this connection. Reopening // it on process restart never reads or overwrites an auth profile. -func bootstrapProfile(path string) (*auth.Profile, error) { +func bootstrapProfile(path string) (*auth.Profile, bool, error) { raw, err := runtimefs.ReadPrivatePath(path, runtimebootstrap.MaxBytes) if err != nil { - return nil, errors.New("connect: Runtime bootstrap file unavailable") + return nil, false, errors.New("connect: Runtime bootstrap file unavailable") } input, err := runtimebootstrap.Decode(raw) if err != nil { - return nil, err + return nil, false, err } if err = localworkspace.Bind(input.EnvironmentID, input.SessionID, input.Workspace, localworkspace.CapabilityDirectory); err != nil { - return nil, errors.New("connect: Runtime workspace binding unavailable") + return nil, false, errors.New("connect: Runtime workspace binding unavailable") } - return &auth.Profile{ServerURL: input.CoreURL, RuntimeID: input.DeviceID, RunnerCredential: input.Credential}, nil + return &auth.Profile{ServerURL: input.CoreURL, RuntimeID: input.DeviceID, RunnerCredential: input.Credential}, input.SuspendControl, nil } diff --git a/apps/daemon/internal/cli/connect_bootstrap_test.go b/apps/daemon/internal/cli/connect_bootstrap_test.go index 32983a8fc..9f899062b 100644 --- a/apps/daemon/internal/cli/connect_bootstrap_test.go +++ b/apps/daemon/internal/cli/connect_bootstrap_test.go @@ -45,7 +45,7 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { } t.Cleanup(localworkspace.Unbind) for range 2 { - p, err := bootstrapProfile(path) + p, _, err := bootstrapProfile(path) if err != nil || p.ServerURL != input.CoreURL || p.RuntimeID != input.DeviceID || p.RunnerCredential != input.Credential { t.Fatal("failed bootstrap/restart", err) } @@ -60,13 +60,13 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(path, []byte("bootstrap-secret"), 0600); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil || strings.Contains(err.Error(), input.Credential) { + if _, _, err = bootstrapProfile(path); err == nil || strings.Contains(err.Error(), input.Credential) { t.Fatal("invalid input fell back or leaked") } if err = os.Remove(path); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil { + if _, _, err = bootstrapProfile(path); err == nil { t.Fatal("missing input fell back to private auth") } } diff --git a/apps/daemon/internal/cli/connect_environment.go b/apps/daemon/internal/cli/connect_environment.go index d30994a06..08c98d930 100644 --- a/apps/daemon/internal/cli/connect_environment.go +++ b/apps/daemon/internal/cli/connect_environment.go @@ -166,7 +166,7 @@ func runEnvironmentConnect(parent context.Context, rc *runContext, profile strin return err } prof := auth.Profile{ServerURL: base, RuntimeID: bound.DeviceID, RunnerCredential: credential} - return rejected(mainLoopRemote(parent, rc, profile, prof, discovery, remote)) + return rejected(mainLoopRemote(parent, rc, profile, prof, discovery, remote, false)) } var ( diff --git a/apps/daemon/internal/cli/suspend_control.go b/apps/daemon/internal/cli/suspend_control.go index b6d302d46..5cbc8312f 100644 --- a/apps/daemon/internal/cli/suspend_control.go +++ b/apps/daemon/internal/cli/suspend_control.go @@ -14,10 +14,9 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/localworkspace" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" ) -const suspendControlEnv = "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE" - type suspendIdentity struct { PID int `json:"pid"` StartTime string `json:"start_time"` @@ -34,14 +33,16 @@ type suspendControl struct { signal chan os.Signal } -func newSuspendControl() (*suspendControl, error) { - path := os.Getenv(suspendControlEnv) - if path == "" { +// newSuspendControl publishes the control at runtimebootstrap.SuspendControlFile +// when the bootstrap enables it, and fails the connection when it cannot. +func newSuspendControl(enabled bool) (*suspendControl, error) { + if !enabled { return nil, nil } - if !filepath.IsAbs(path) || filepath.Clean(path) != path { - return nil, errors.New("connect: absolute suspend control file required") - } + return openSuspendControl(runtimebootstrap.SuspendControlFile) +} + +func openSuspendControl(path string) (*suspendControl, error) { info, err := os.Lstat(filepath.Dir(path)) if err != nil || !info.IsDir() || info.Mode().Perm()&0077 != 0 { return nil, errors.New("connect: private suspend control directory required") diff --git a/apps/daemon/internal/cli/suspend_control_linux_test.go b/apps/daemon/internal/cli/suspend_control_linux_test.go index eda9dcff7..53d890936 100644 --- a/apps/daemon/internal/cli/suspend_control_linux_test.go +++ b/apps/daemon/internal/cli/suspend_control_linux_test.go @@ -21,13 +21,12 @@ func TestResumeControlRejectsStaleIdentityAndDiscardsPreParkSignals(t *testing.T t.Fatal(err) } path := filepath.Join(dir, "control.json") - t.Setenv(suspendControlEnv, path) environment := uuid.NewString() if err := localworkspace.Bind(environment, uuid.NewString(), t.TempDir(), t.TempDir()); err != nil { t.Fatal(err) } t.Cleanup(localworkspace.Unbind) - control, err := newSuspendControl() + control, err := openSuspendControl(path) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/cli/suspend_process_other.go b/apps/daemon/internal/cli/suspend_process_other.go index 5d57afa2d..3b7a69155 100644 --- a/apps/daemon/internal/cli/suspend_process_other.go +++ b/apps/daemon/internal/cli/suspend_process_other.go @@ -9,8 +9,6 @@ import ( "os" ) -const suspendControlEnv = "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE" - type suspendIdentity struct { EnvironmentID string SuspendID string @@ -21,8 +19,8 @@ type suspendControl struct { lastResumed *proto.EnvironmentSuspendPayload } -func newSuspendControl() (*suspendControl, error) { - if os.Getenv(suspendControlEnv) != "" { +func newSuspendControl(enabled bool) (*suspendControl, error) { + if enabled { return nil, errors.New("hosted suspension requires Linux") } return nil, nil diff --git a/apps/web/e2e/fixture-console.mjs b/apps/web/e2e/fixture-console.mjs index e2fa694f3..a6c92d720 100644 --- a/apps/web/e2e/fixture-console.mjs +++ b/apps/web/e2e/fixture-console.mjs @@ -82,18 +82,18 @@ function nodeRollout(previous = 0) { } function unconfiguredDeployment(generation = 0, ownerEpoch = 3) { - return { installation_id: INSTALLATION_ID, provider: "", credential_configured: false, core_url: publicUrl(), reset: null, rollout: noNodeRollout(), owner_epoch: ownerEpoch, generation, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "", credential_configured: false, core_url: publicUrl(), reset: null, rollout: noNodeRollout(), owner_epoch: ownerEpoch, generation, mode: "", resources: { allocations: 0, running: 0, pending: 0 }, suspension: null }; } function configuredDeployment() { - return { installation_id: INSTALLATION_ID, provider: "docker", credential_configured: false, configuration: {}, metadata: {}, core_url: publicUrl(), reset: null, rollout: nodeRollout(), owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 } }, specification_digest: "fixture", suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "docker", credential_configured: false, configuration: {}, metadata: {}, core_url: publicUrl(), reset: null, rollout: nodeRollout(), owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, running: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 } }, specification_digest: "fixture", suspension: null }; } /** The E2B template build as Core read it when the selection was saved. */ const templateBuild = { status: "ready", resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 10240 } }; // E2B runs sandboxes in its cloud: no nodes, only what Core holds there. function e2bDeployment() { - return { ...configuredDeployment(), provider: "e2b", mode: "direct", rollout: noNodeRollout(), resources: { allocations: 3, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, configuration: { template: "oac-runtime:0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b", api_url: "https://api.e2b.app", domain: "e2b.app" } , credential_configured: true, metadata: { template_build: templateBuild } }; + return { ...configuredDeployment(), provider: "e2b", mode: "direct", rollout: noNodeRollout(), resources: { allocations: 3, running: 3, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, configuration: { template: "oac-runtime:0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b", api_url: "https://api.e2b.app", domain: "e2b.app" } , credential_configured: true, metadata: { template_build: templateBuild } }; } function reset(mode = "login", fresh = false, sandbox = "configured", nodes = "demo", address = "public", installers = true, artifacts = "docker,microsandbox") { @@ -419,7 +419,7 @@ async function sandboxRoute(request, response, path, url) { if (!initialize && !e2b) { for (const node of state.nodes) node.rollout = { state: node.online ? "preparing" : "unknown", ready_generation: node.rollout.ready_generation }; } - const held = initialize ? { allocations: 0, pending: 0 } : previous.resources; + const held = initialize ? { allocations: 0, running: 0, pending: 0 } : previous.resources; state.deployment = { ...configuredDeployment(), provider: input.provider, mode: e2b ? "direct" : "nodes", generation: previous.generation + 1, owner_epoch: previous.owner_epoch, diff --git a/apps/web/e2e/nodes.spec.ts b/apps/web/e2e/nodes.spec.ts index 0bc1c57d8..bb0889aa6 100644 --- a/apps/web/e2e/nodes.spec.ts +++ b/apps/web/e2e/nodes.spec.ts @@ -315,7 +315,7 @@ test("shows the retained E2B build while a replacement key is checked", async ({ test("edits only the saved backend, preserving a custom size", async ({ page, request }) => { const current = { resources: { cpus: 7, memory_mib: 8192 } }; let deployment = { configuration: {}, metadata: {}, credential_configured: false, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", reset: null, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 0, preparing: 0, failed: 0, unknown: 0 } }, - owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: current, specification_digest: "e".repeat(64), + owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, running: 0, pending: 0 }, specification: current, specification_digest: "e".repeat(64), suspension: null }; let submitted: Record | null = null; await page.route("**/core/v1/sandbox/deployment", async (route) => { diff --git a/apps/web/e2e/sandbox-generation.spec.ts b/apps/web/e2e/sandbox-generation.spec.ts index 612823d57..a94ac1432 100644 --- a/apps/web/e2e/sandbox-generation.spec.ts +++ b/apps/web/e2e/sandbox-generation.spec.ts @@ -58,7 +58,7 @@ test("online configuration retains existing resources and stops rapid polling wh await expect(rollout(page)).toBeVisible(); const before = await inventory(page); expect(before.allocations.length).toBeGreaterThan(0); - await setDeployment(request, { resources: { allocations: before.allocations.length, pending: 0 } }); + await setDeployment(request, { resources: { allocations: before.allocations.length, running: before.allocations.length, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); await page.getByRole("button", { name: "Change resources", exact: true }).click(); await page.getByRole("button", { name: /^Large/ }).click(); diff --git a/apps/web/e2e/sandbox-reset.spec.ts b/apps/web/e2e/sandbox-reset.spec.ts index cb6c1e802..ca6cd7086 100644 --- a/apps/web/e2e/sandbox-reset.spec.ts +++ b/apps/web/e2e/sandbox-reset.spec.ts @@ -41,7 +41,7 @@ test("auto reset requires a bounded deadline, then preserves Core progress past await page.setViewportSize({ width: 1280, height: 1000 }); await page.emulateMedia({ colorScheme: "light", reducedMotion: "reduce" }); await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 3, pending: 1 } }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); await expect(dialog).toBeVisible(); @@ -71,7 +71,7 @@ test("auto reset requires a bounded deadline, then preserves Core progress past expect((await submitted).postDataJSON()).toEqual({ expected_generation: 1, clear: "auto", deadline_seconds: 300 }); await expect(progress(page)).toBeVisible(); // These timestamps are in the past: the browser still cannot invent force or completion. - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); await expect(progress(page)).toContainText("offline-build-worker"); await expect(progress(page)).toContainText("Busy work can finish until the deadline."); @@ -83,7 +83,7 @@ test("auto reset requires a bounded deadline, then preserves Core progress past test("escalation and cancellation require explicit confirmation and never undo previous clearing", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.reload(); await progress(page).getByRole("button", { name: "Force reset now" }).click(); let confirm = page.getByRole("dialog", { name: "Force reset now?" }); @@ -110,7 +110,7 @@ test("escalation and cancellation require explicit confirmation and never undo p test("force reset requires confirmation and reconfiguration uses the completed generation once", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); await dialog.getByLabel("Force — cancel remaining work now", { exact: true }).check(); @@ -134,7 +134,7 @@ test("force reset requires confirmation and reconfiguration uses the completed g test("an applied reset with a lost response stays blocked through failed reads and is never replayed", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); let failReads = false; @@ -165,7 +165,7 @@ test("an applied reset with a lost response stays blocked through failed reads a test("a stale reset is refused until fresh state is reviewed, with no automatic resubmission", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); - await setDeployment(request, { generation: 2, resources: { allocations: 3, pending: 1 } }); + await setDeployment(request, { generation: 2, resources: { allocations: 3, running: 3, pending: 1 } }); const dialog = await openReset(page); const staleSubmission = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith(resetPath)); const freshRead = page.waitForResponse(async (received) => received.request().method() === "GET" @@ -195,7 +195,7 @@ test("reset progress survives failed node and deployment reads with visible qual await page.setViewportSize({ width: 1280, height: 1000 }); await page.emulateMedia({ colorScheme: "dark" }); await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.route("**/core/v1/sandbox/nodes", (route) => route.fulfill(unavailable)); await page.reload(); await expect(progress(page)).toContainText("offline-build-worker"); @@ -219,7 +219,7 @@ test("reset progress survives failed node and deployment reads with visible qual test("a pending reset survives navigation and a lost response cannot reopen stale write controls", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); let accepted!: () => void; @@ -281,7 +281,7 @@ test("Nodes and configuration adopt reset and completion learned on Overview bef await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); const cachedAt = Date.now(); await page.getByRole("button", { name: "Overview", exact: true }).click(); - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.getByRole("button", { name: "Refresh", exact: true }).click(); await expect(page.getByRole("heading", { name: "Sandbox reset in progress" })).toBeVisible(); await expectEnrollmentBlocked(page); @@ -292,7 +292,7 @@ test("Nodes and configuration adopt reset and completion learned on Overview bef // No refresh click: shared active reset starts bounded read-only polling on configuration. const forced = observedReset("force"); forced.remaining = { busy: 0, idle: 0, cleanup: 1, on_offline_nodes: 1, offline_nodes: [{ node_id: "offline-owned", name: "offline-build-worker", resources: 1 }] }; - await setDeployment(request, { resources: { allocations: 1, pending: 0 }, reset: forced }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 }, reset: forced }); await expect(progress(page)).toContainText("Remaining hosted Sessions are being archived", { timeout: 12_000 }); await expect(progress(page).getByRole("button", { name: "Force reset now" })).toHaveCount(0); @@ -314,7 +314,7 @@ test("Nodes and configuration adopt reset and completion learned on Overview bef test("signing out clears pending reset ownership and a late response cannot overwrite the new login", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); let accepted!: () => void; diff --git a/apps/web/e2e/sandbox-rollout-details.spec.ts b/apps/web/e2e/sandbox-rollout-details.spec.ts index 4695ac5ee..43d5d2ec7 100644 --- a/apps/web/e2e/sandbox-rollout-details.spec.ts +++ b/apps/web/e2e/sandbox-rollout-details.spec.ts @@ -9,7 +9,7 @@ test("keeps rollout summary brief and opens all Core observations in details", a await page.route("**/core/v1/sandbox/deployment", async (route) => { const response = await route.fetch(); const deployment = await response.json(); - await route.fulfill({ response, json: { ...deployment, generation: 4, resources: { allocations: 8, pending: 0 }, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } } }); + await route.fulfill({ response, json: { ...deployment, generation: 4, resources: { allocations: 8, running: 8, pending: 0 }, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } } }); }); await openConsole(page, request, "settings?id=sandbox"); const summary = page.getByRole("region", { name: "Configuration rollout", exact: true }); diff --git a/apps/web/e2e/unified-settings.spec.ts b/apps/web/e2e/unified-settings.spec.ts index 8b4e17c33..549e8495e 100644 --- a/apps/web/e2e/unified-settings.spec.ts +++ b/apps/web/e2e/unified-settings.spec.ts @@ -118,7 +118,7 @@ test("settings content width does not constrain the reset dialog backdrop", asyn test("reset progress actions stay inside settings on narrow screens", async ({ page, request }) => { await page.setViewportSize({ width: 375, height: 844 }); await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 1, pending: 0 }, reset: { clear: "auto", requested_at: "2026-10-13T08:00:00Z", deadline_at: "2026-10-13T09:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 }, reset: { clear: "auto", requested_at: "2026-10-13T08:00:00Z", deadline_at: "2026-10-13T09:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }); await page.getByRole("button", { name: "Refresh sandbox state" }).click(); const region = page.getByRole("region", { name: "Reset in progress" }); await expect(region).toBeVisible(); diff --git a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx index 0cb7598a1..79341708d 100644 --- a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx +++ b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx @@ -6,7 +6,7 @@ import { SandboxResetNotice } from "./SandboxResetNotice"; const reset: SandboxReset = { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 2, on_offline_nodes: 1, offline_nodes: [{ node_id: "n1", name: "Node 1", resources: 1 }] } }; -const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, pending: 0 }, suspension: null }); +const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, running: 3, pending: 0 }, suspension: null }); const render = (value: SandboxDeployment | undefined, failed = false) => renderToStaticMarkup( {}} />); describe("reset notices on read-only surfaces", () => { diff --git a/apps/web/src/features/fleet/fleet-queries.test.ts b/apps/web/src/features/fleet/fleet-queries.test.ts index e0eb11cc1..aabdcc5c4 100644 --- a/apps/web/src/features/fleet/fleet-queries.test.ts +++ b/apps/web/src/features/fleet/fleet-queries.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxScope } from "../sandbox/sandbox-queries"; import { fleetQuery } from "./fleet-queries"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null, +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, running: 1, pending: 0 }, suspension: null, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); diff --git a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx index 756ae7660..49a8ba840 100644 --- a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx +++ b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx @@ -10,7 +10,7 @@ import { fleetQuery, type FleetSnapshot } from "./fleet-queries"; import { FleetReadNotice } from "./FleetReadNotice"; import { fleetSnapshot, useSandboxFleet } from "./use-sandbox-fleet"; -const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, running: 1, pending: 0 }, suspension: null }; function Probe() { const { state, deployment } = useSandboxFleet(); @@ -33,7 +33,7 @@ function render(latest: SandboxDeployment, previous = configured, failed = false describe("fleet evidence after deployment changes", () => { it("cannot complete onboarding or show old resource counts when reset leaves an unconfigured deployment", () => { const resetting = { ...configured, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } } satisfies SandboxDeployment; - const completed = { ...configured, provider: "", mode: "", generation: 2, resources: { allocations: 0, pending: 0 } } satisfies SandboxDeployment; + const completed = { ...configured, provider: "", mode: "", generation: 2, resources: { allocations: 0, running: 0, pending: 0 } } satisfies SandboxDeployment; const html = render(completed, resetting); expect(html).toContain('data-fleet="loading"'); expect(html).toContain('data-sandbox="checking"'); diff --git a/apps/web/src/features/metrics/SandboxMetricsPage.tsx b/apps/web/src/features/metrics/SandboxMetricsPage.tsx index d63f84329..b0f5160d4 100644 --- a/apps/web/src/features/metrics/SandboxMetricsPage.tsx +++ b/apps/web/src/features/metrics/SandboxMetricsPage.tsx @@ -145,7 +145,7 @@ function CloudSection({ deployment }: { deployment: SandboxDeployment }) { >

{t("sandbox.cloud.counts")}

- + ): string { function cloudHost(fleet: FleetSnapshot | null): CloudHost | null { if (fleet?.deployment.mode !== "direct") return null; - return { running: fleet.deployment.resources.allocations, pending: fleet.deployment.resources.pending }; + return { running: fleet.deployment.resources.running, pending: fleet.deployment.resources.pending }; } /** Core and its sandbox nodes (or the Provider's cloud) as an inventory; each row opens its page. */ diff --git a/apps/web/src/features/overview/getting-started.test.ts b/apps/web/src/features/overview/getting-started.test.ts index 0409d1a5f..aaffd5da0 100644 --- a/apps/web/src/features/overview/getting-started.test.ts +++ b/apps/web/src/features/overview/getting-started.test.ts @@ -7,7 +7,7 @@ import { node, project } from "./test-fixtures"; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", - resources: { allocations: 0, pending: 0 }, suspension: null, ...overrides, + resources: { allocations: 0, running: 0, pending: 0 }, suspension: null, ...overrides, }); const fleet = (value: SandboxDeployment, nodes = [node("n1")]): FleetState => ({ status: "ready", snapshot: { deployment: value, nodes, allocations: [], loadedAt: 0 }, targetGeneration: value.generation, refreshing: false, error: null }); const sandboxes = (state: FleetState) => gettingStartedSteps({ sandboxReset: false, fleet: state, localOnly: false, projects: [], sessions: 0, harnesses: [] }).sandboxes; diff --git a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx index c5184de13..f8678adae 100644 --- a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx +++ b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx @@ -5,7 +5,7 @@ import { SandboxResetControls } from "./SandboxResetControls"; const reset: SandboxReset = { clear: "auto", requested_at: "2020-01-01T10:00:00Z", deadline_at: "2020-01-01T11:00:00Z", forced_at: null, remaining: { busy: 2, idle: 1, cleanup: 3, on_offline_nodes: 2, offline_nodes: [{ node_id: "offline-a", name: "Offline A", resources: 2 }] } }; -const deployment = (value: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, pending: 1 }, suspension: null }); +const deployment = (value: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, running: 5, pending: 1 }, suspension: null }); const render = (value: SandboxReset | null, stale = false) => renderToStaticMarkup( true} onCancel={async () => true} />); describe("authoritative reset progress", () => { diff --git a/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx b/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx index 0364ada21..732cc1cf7 100644 --- a/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx +++ b/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx @@ -7,7 +7,7 @@ import { SandboxRolloutSummary } from "./SandboxRolloutSummary"; import { NodeRolloutStatus } from "./NodeRolloutStatus"; import { SandboxDeploymentSettings } from "./SandboxDeploymentSettings"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, installation_id: "i", owner_epoch: 1, generation: 4, provider: "docker", mode: "nodes", core_url: "https://core.example", resources: { allocations: 8, pending: 2 }, reset: null, suspension: null, +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, installation_id: "i", owner_epoch: 1, generation: 4, provider: "docker", mode: "nodes", core_url: "https://core.example", resources: { allocations: 8, running: 8, pending: 2 }, reset: null, suspension: null, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } }; describe("authoritative configuration rollout", () => { diff --git a/apps/web/src/features/sandbox/SuspensionSettings.test.tsx b/apps/web/src/features/sandbox/SuspensionSettings.test.tsx index db09a309f..f67a766e9 100644 --- a/apps/web/src/features/sandbox/SuspensionSettings.test.tsx +++ b/apps/web/src/features/sandbox/SuspensionSettings.test.tsx @@ -11,6 +11,8 @@ describe("suspension settings", () => { expect(html).toContain('id="sandbox-idle-seconds"'); expect(html).toContain('value="300"'); expect(html).toContain("24 hours"); + expect(html).toContain("Edit node"); + expect(renderToStaticMarkup( true} />)).not.toContain("Edit node"); expect(renderToStaticMarkup( true} />)).toBe(""); }); }); diff --git a/apps/web/src/features/sandbox/SuspensionSettings.tsx b/apps/web/src/features/sandbox/SuspensionSettings.tsx index 37404433e..15232b677 100644 --- a/apps/web/src/features/sandbox/SuspensionSettings.tsx +++ b/apps/web/src/features/sandbox/SuspensionSettings.tsx @@ -12,7 +12,7 @@ import { writeSandboxDeployment } from "./sandbox-deployment-write"; function RetentionHelp({ deployment }: { deployment: SandboxDeployment }) { const { t } = useTranslation("sandbox"); - return <>{t("Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume; capacity and a compatible node must be available.")}{deployment.specification?.workspace ? <> {t("Snapshot expiry releases memory state. With independent workspace storage, files and conversation remain available for a cold continuation; processes restart.")} : null}; + return <>{t(deployment.mode === "direct" ? "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume." : "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume; capacity and a compatible node must be available.")}{deployment.specification?.workspace ? <> {t("Snapshot expiry releases memory state. With independent workspace storage, files and conversation remain available for a cold continuation; processes restart.")} : null}; } /** Settings edits the deployment-wide idle time; each node's limits are on Nodes. */ @@ -41,10 +41,10 @@ export function SuspensionForm({ deployment, disabled, onSave }: { deployment: S if (!policy) return null; const idle = draft ?? String(policy.idle_seconds); const valid = /^\d+$/.test(idle) && Number(idle) >= 30 && Number(idle) <= 86400; - return
+ return
{ event.preventDefault(); if (valid && !disabled && await onSave(Number(idle))) setDraft(null); }}> -
{t("Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged. Capacity pressure may suspend an idle sandbox after 15 seconds.")}
+
{t("Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged.")}{deployment.mode === "direct" ? null : <> {t("Capacity pressure may suspend an idle sandbox after 15 seconds.")}}
setDraft(event.target.value)} aria-invalid={!valid} aria-describedby={!valid ? "sandbox-idle-error" : undefined} /> diff --git a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts index be04e2036..d1066ddd3 100644 --- a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts +++ b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts @@ -5,7 +5,7 @@ import { sandboxAdmin, sandboxDeploymentQuery } from "./sandbox-queries"; import { writeSandboxDeployment } from "./sandbox-deployment-write"; import { sandboxWriteOwnershipQuery } from "./sandbox-write-ownership"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null }; +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, running: 0, pending: 0 }, suspension: null }; function deferred() { let resolve!: (value: T) => void; let reject!: (error: Error) => void; diff --git a/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx b/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx index d9842bc34..dddbf62b0 100644 --- a/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx +++ b/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx @@ -18,7 +18,7 @@ function cache(provider: SandboxDeployment["provider"]) { const client = new QueryClient({ defaultOptions: { queries: { enabled: false, retry: false, gcTime: Infinity } } }); const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, installation_id: "install", owner_epoch: 1, generation: 2, provider, core_url: "https://core.example", - mode: provider === "" ? "" : provider === "e2b" ? "direct" : "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null, + mode: provider === "" ? "" : provider === "e2b" ? "direct" : "nodes", reset: null, resources: { allocations: 0, running: 0, pending: 0 }, suspension: null, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, }; const config: SandboxConsoleConfig = { node_installer: false, node_installer_sha256: "", node_artifacts: [] }; diff --git a/apps/web/src/features/sandbox/sandbox-queries.test.ts b/apps/web/src/features/sandbox/sandbox-queries.test.ts index 62bef4afb..08854805d 100644 --- a/apps/web/src/features/sandbox/sandbox-queries.test.ts +++ b/apps/web/src/features/sandbox/sandbox-queries.test.ts @@ -10,7 +10,7 @@ const reset: SandboxReset = { }; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "installation-a", provider: "docker", core_url: "https://core.example", reset, - owner_epoch: 1, generation: 2, mode: "nodes", resources: { allocations: 2, pending: 0 }, suspension: null, ...overrides, + owner_epoch: 1, generation: 2, mode: "nodes", resources: { allocations: 2, running: 2, pending: 0 }, suspension: null, ...overrides, }); const clients: QueryClient[] = []; const cache = () => { const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: Infinity } } }); clients.push(client); return client; }; diff --git a/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx b/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx index 63c8667c2..ae16d744a 100644 --- a/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx +++ b/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx @@ -6,7 +6,7 @@ import { node } from "../overview/test-fixtures"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxSnapshotQuery } from "./sandbox-queries"; import { useSandboxPageState } from "./use-sandbox-page-state"; -const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, running: 1, pending: 0 }, suspension: null }; const resetting: SandboxDeployment = { ...configured, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; const clients: QueryClient[] = []; function cache() { diff --git a/apps/web/src/i18n/locales/en/metrics.ts b/apps/web/src/i18n/locales/en/metrics.ts index e36b7d79a..4ada3db8e 100644 --- a/apps/web/src/i18n/locales/en/metrics.ts +++ b/apps/web/src/i18n/locales/en/metrics.ts @@ -189,11 +189,11 @@ export const metrics = { }, cloud: { title: "Provider cloud", - counts: "The running count comes from Core’s allocations in the provider’s cloud. The hosted Runtime total below counts Core’s hosted observation records across projects, including every reported lifecycle state. These sources have different coverage and refresh separately.", + counts: "The running count comes from Core’s running allocations in the provider’s cloud; suspended sandboxes are not counted. The hosted Runtime total below counts Core’s hosted observation records across projects, including every reported lifecycle state. These sources have different coverage and refresh separately.", help: "This deployment runs its sandboxes in the provider's cloud: there are no machines to manage. Counts are Core's own records of what it placed there.", manage: "Sandbox backend", running: "Running sandboxes", - runningHelp: "Sandboxes Core currently holds in the provider’s cloud.", + runningHelp: "Sandboxes running in the provider’s cloud for Core.", pending: "Starting or changing", size: "Each sandbox", }, diff --git a/apps/web/src/i18n/locales/zh-CN/metrics.ts b/apps/web/src/i18n/locales/zh-CN/metrics.ts index b6bf73549..4d9405e28 100644 --- a/apps/web/src/i18n/locales/zh-CN/metrics.ts +++ b/apps/web/src/i18n/locales/zh-CN/metrics.ts @@ -189,11 +189,11 @@ export const metrics = { }, cloud: { title: "提供商云端", - counts: "运行数量来自 Core 在提供商云端的分配记录;下方托管 Runtime 总数统计各项目的托管观测记录,包含所有已报告的生命周期状态。两者覆盖范围不同,并分别刷新。", + counts: "运行数量来自 Core 在提供商云端正在运行的分配记录,不含已挂起的沙箱;下方托管 Runtime 总数统计各项目的托管观测记录,包含所有已报告的生命周期状态。两者覆盖范围不同,并分别刷新。", help: "这个部署的沙箱在提供商的云端运行,没有需要管理的机器。数量是 Core 自己记录的、放在云端的沙箱。", manage: "沙箱提供商", running: "运行中的沙箱", - runningHelp: "Core 当前在提供商云端占用的沙箱。", + runningHelp: "Core 在提供商云端正在运行的沙箱。", pending: "启动或切换中", size: "每个沙箱", }, diff --git a/apps/web/src/i18n/locales/zh-CN/sandbox.ts b/apps/web/src/i18n/locales/zh-CN/sandbox.ts index 7f2768737..116629d96 100644 --- a/apps/web/src/i18n/locales/zh-CN/sandbox.ts +++ b/apps/web/src/i18n/locales/zh-CN/sandbox.ts @@ -5,9 +5,12 @@ export const sandbox = { "Reset": "重置", "Clear this configuration to choose a different backend. Hosted Sessions are archived permanently.": "清除当前配置后才能选择其他沙箱提供商。托管 Session 将被永久归档。", "Idle before suspension (seconds)": "空闲多久后挂起(秒)", - "Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged. Capacity pressure may suspend an idle sandbox after 15 seconds.": "整个部署统一生效:30–86,400 秒。适用于现有和新建的运行中沙箱,无需重启。已经开始的挂起和快照保留期不变。容量紧张时,沙箱空闲 15 秒后可能提前挂起。", + "Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged.": "整个部署统一生效:30–86,400 秒。适用于现有和新建的运行中沙箱,无需重启。已经开始的挂起和快照保留期不变。", + "Capacity pressure may suspend an idle sandbox after 15 seconds.": "容量紧张时,沙箱空闲 15 秒后可能提前挂起。", "Enter a whole number from 30 to 86,400.": "请输入 30 到 86,400 之间的整数。", "Automatic suspension": "自动挂起", + "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume.": "这是新挂起任务采用的当前策略,不是现有快照的到期倒计时。下一轮 Turn 会请求恢复。", + "The idle policy applies to the whole deployment.": "空闲策略对整个部署生效。", "Snapshot retention": "快照保留期", "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume; capacity and a compatible node must be available.": "这是新挂起任务采用的当前策略,不是现有快照的到期倒计时。下一轮 Turn 会请求恢复,需要可用容量及兼容节点。", "Snapshot expiry releases memory state. With independent workspace storage, files and conversation remain available for a cold continuation; processes restart.": "快照到期后释放内存状态。使用独立工作区存储时,文件和对话仍可用于冷启动续接,进程会重新启动。", diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index e0d305ff2..967570464 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -1637,9 +1637,12 @@ definitions: type: integer pending: type: integer + running: + type: integer required: - allocations - pending + - running type: object deployment.Rollout: properties: diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index c6b2972b1..a7a7e997a 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -79,6 +79,7 @@ Each operation carries its own arguments and returns the following result on suc | `initial` | `Initial` | None | `compute` | | `new_compute` | `NewCompute` | Positive compute `generation` and optional `snapshot` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | None | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` and `command` | `command` | diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 50f0307af..22f6c0740 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -78,7 +78,7 @@ GET and successful writes return `installation_id`, `provider`, `core_url` (read - `metadata.template_build` is `{status, resources: {cpus, memory_mib, root_disk_mib}}`: the build as Core read it through the pinned SDK when the selection was saved. GET never calls E2B, so it stays cheap during an E2B outage. Validation admits only a `ready` build whose CPU count and memory equal the selected values; `root_disk_mib` is the build's native disk size, which Core does not enforce. Unknown values are null, `metadata: {}` means no observation was recorded, and an identical PUT without a credential does not refresh it. - `suspension` is `{idle_seconds, retention_seconds}`, Core's [suspension policy](../../docs/sandbox-provider.md#suspension), when the selected Provider declares checkpoint support; any other deployment, including an unconfigured one, returns null. -- Request `resources` and response `specification.resources` are per-sandbox limits. Response `resources.allocations` and `resources.pending` count unreleased allocations and pending hosted Environments without an allocation. +- Request `resources` and response `specification.resources` are per-sandbox limits. Response `resources.allocations` and `resources.pending` count unreleased allocations and pending hosted Environments without an allocation; `resources.running` counts the unreleased allocations whose compute is running, so it excludes suspended ones. - An unconfigured deployment has an empty provider and no specification. Docker and microsandbox use `mode: nodes`; E2B uses `mode: direct`, without a synthetic node. - `generation` identifies the saved selection. `owner_epoch` fences the execution owner and node connections; it does not replace `expected_generation`. - `specification_digest` is the server's identity of the provider, limits and derived workspace capability receipt; enrollment echoes it unchanged. @@ -220,7 +220,7 @@ Some fields keep one name across providers but differ in meaning, or do not appl | Independent workspace filesystem | Required: [Kernel NFS adapter](../../docs/workspace-provider.md#kernel-nfs-adapter), configured separately from compute | Unsupported | Unsupported | | Deployment `specification.resources` | `cpus`, `memory_mib`, `root_disk_mib` and, as a quota request, `environment_disk_mib` | `cpus` and `memory_mib`, equal to the ready template build's and taken from it when omitted; no disk fields | `cpus` and `memory_mib`; no disk quota | | Deployment `metadata.template_build` | Absent: `metadata` is empty | The build as Core read it when the selection was saved | Absent: `metadata` is empty | -| Deployment `suspension` | `{idle_seconds, retention_seconds}` | `null`; Core does not suspend E2B sandboxes | `null` | +| Deployment `suspension` | `{idle_seconds, retention_seconds}` | `{idle_seconds, retention_seconds}`; Core pauses an idle E2B sandbox with the SDK's native pause | `null` | | Deployment `resources.allocations`, `resources.pending` | Totals across all nodes | Core's unreleased E2B sandboxes, and hosted Environments waiting for one | Totals across all nodes | | Enrollment-token `max_active`, `max_retained` | Both limits apply; an omitted limit is derived at enrollment | 409 `sandbox_deployment_conflict`, after the 400 capacity checks; E2B has no nodes | `max_retained` always equals `max_active` | | Node list and detail | Enrolled nodes | Empty list; detail returns 404 | Enrolled nodes | diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index ac649c21d..64dd5a810 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 090803f51d2e29d8900cfe8612b0eb21363a0fd2687daf3d368cc746dd65809f +source_hash: 3648a80013f1ae90031430c5bdf28d6ee5b59f55dda4b168dfeb99ab5bcec3e2 --- 沙箱节点协议连接 Core 与 Docker 或 microsandbox 主机:主机针对其本地提供商执行提供商操作,并报告就绪状态、主机测量值及其持有的部署代次。 @@ -81,6 +81,7 @@ Core 发送包含以下内容的 `request` 帧: | `initial` | `Initial` | 无 | `compute` | | `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `snapshot` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | 无 | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` 和 `command` | `command` | diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index 306cefcb9..8ddf657b2 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: 4b77c11fe9a80e4b6d200ca161ab53e006bfc46b185262ee858fc65d75b5112a +source_hash: 673ae35810faa99f2a3fa480f6b487b4b8a3e7a21a47a4b1fa9c22df6bc53845 --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择沙箱提供商和每个沙箱的资源。PostgreSQL 为每个安装维护一个当前有效选择;Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径,且不能覆盖其资源。Runtime 不属于该选择:每个节点运行其所属安装验证过的 Runtime。该选择独立于 Harness。部署可以保持未配置状态,没有节点;此时它拒绝托管准入。 @@ -81,7 +81,7 @@ GET 和成功的写入操作会返回 `installation_id`、`provider`、`core_url - `metadata.template_build` 为 `{status, resources: {cpus, memory_mib, root_disk_mib}}`:这是保存选择时 Core 通过固定版本 SDK 读取的构建。GET 绝不会调用 E2B,因此 E2B 中断期间该操作仍保持低成本。验证仅接受 CPU 数量和内存与所选值一致的 `ready` 构建;`root_disk_mib` 是构建的原生磁盘大小,Core 不会强制执行该值。未知值为 null,`metadata: {}` 表示未记录任何观测,在不提供凭据的情况下提交完全相同的 PUT 也不会刷新它。 - 所选提供商声明 checkpoint 支持时,`suspension` 为 `{idle_seconds, retention_seconds}`,即 Core 的 [suspension policy](../../../docs/zh/sandbox-provider.md#suspension);其他部署(包括未配置的部署)返回 null。 -- 请求中的 `resources` 和响应中的 `specification.resources` 是每个沙箱的限制。响应中的 `resources.allocations` 和 `resources.pending` 分别计算尚未释放的分配,以及尚未分配沙箱的待处理托管 Environment。 +- 请求中的 `resources` 和响应中的 `specification.resources` 是每个沙箱的限制。响应中的 `resources.allocations` 和 `resources.pending` 分别计算尚未释放的分配,以及尚未分配沙箱的待处理托管 Environment;`resources.running` 计算计算资源正在运行的未释放分配,因此不含已挂起的分配。 - 未配置的部署具有空的提供商且没有 specification。Docker 和 microsandbox 使用 `mode: nodes`;E2B 使用 `mode: direct`,且没有合成节点。 - `generation` 标识已保存的选择。`owner_epoch` 用于对执行所有者和节点连接进行栅栏隔离;它不能替代 `expected_generation`。 - `specification_digest` 是服务器对提供商、限制和派生的工作区能力回执的标识;注册过程会原样回显该值。 @@ -225,7 +225,7 @@ Core 用一条规则,从节点主机和单个沙箱的大小推导推荐值。 | 独立工作区文件系统 | 必需:[Kernel NFS adapter](../../../docs/zh/workspace-provider.md#kernel-nfs-adapter),与计算资源分开配置 | 不支持 | 不支持 | | 部署 `specification.resources` | `cpus`、`memory_mib`、`root_disk_mib`,以及作为配额请求的 `environment_disk_mib` | `cpus` 和 `memory_mib`,必须等于就绪模板构建中的值;省略时取自该构建;无磁盘字段 | `cpus` 和 `memory_mib`;无磁盘配额 | | 部署 `metadata.template_build` | 不存在:`metadata` 为空 | Core 保存选择时读取到的构建 | 不存在:`metadata` 为空 | -| 部署 `suspension` | `{idle_seconds, retention_seconds}` | `null`;Core 不暂停 E2B 沙箱 | `null` | +| 部署 `suspension` | `{idle_seconds, retention_seconds}` | `{idle_seconds, retention_seconds}`;Core 用 SDK 的原生 pause 暂停空闲的 E2B 沙箱 | `null` | | 部署 `resources.allocations`、`resources.pending` | 所有节点的总数 | Core 中尚未释放的 E2B 沙箱,以及正在等待沙箱的托管 Environment | 所有节点的总数 | | 注册令牌 `max_active`、`max_retained` | 两个限制均适用;省略的限制在注册时推导 | 先执行 400 容量检查,然后返回 409 `sandbox_deployment_conflict`;E2B 没有节点 | `max_retained` 始终等于 `max_active` | | 节点列表和详情 | 已注册节点 | 空列表;详情返回 404 | 已注册节点 | diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index ac302218b..5fd98fa34 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -53,11 +53,16 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `environment_id` | Canonical nonzero UUID of the Environment the Runtime serves | | `session_id` | Canonical nonzero UUID of the Session whose native history the Runtime keeps | | `workspace` | Clean absolute directory of the workspace, `runtimebootstrap.RuntimeWorkspace` in the packaged Runtime layout. The Runtime serves it as `runtimebootstrap.PublicWorkspace` (`/workspace`), the path applications and Core use | +| `suspend_control` | Required boolean. Core sets it exactly when the Provider declares [checkpoint support](./sandbox-provider.md), and the Provider forwards it unchanged | The decoder rejects unknown, duplicate, missing and case-aliased fields, other versions and documents larger than `runtimebootstrap.MaxBytes`. Errors never include submitted values. A missing or malformed file fails before the daemon connects. The file is the only authentication input for this launch: the daemon refuses to combine it with self-hosted enrollment options, and reads the credential into memory without saving it to a stored profile. Credentials never go in command arguments, environment variables or receipts. The provider keeps the file for process restarts and removes it only during explicit cleanup of the resources it owns. +## Suspend control + +When `suspend_control` is true, the daemon publishes its suspend control at `runtimebootstrap.SuspendControlFile` and parks for suspension. Core's wake command, `oac-daemon resume --control-file`, reads that file. A Provider that declares checkpoint support prepares the file's directory before it starts the daemon: private to the Runtime account (mode 0700, owned by UID 1000). `oac-daemon connect` fails when `suspend_control` is true and it cannot publish the control, and the control is never enabled by any other setting. + ## Timeouts and limits | Value | Applies to | On breach | diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index ca101398a..f47f4f749 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -52,7 +52,7 @@ Isolation belongs to the provider's infrastructure, not the daemon; see [Runtime A backend without a native renewable lease, such as Docker, still keeps Core's hosted expiry and cleanup requirements. Every adapter implements `RunCommand` and its tests exercise it, but Core's orchestration does not call it; only the node transport forwards it. Confidential command input travels in `Command.Stdin`, never in arguments or logs, and the result keeps byte order, bounded output and the actual exit status. -`Bootstrap.Workspace` and `ResumeRequest.Workspace` carry an external workspace binding exactly when the Provider's `DeploymentPolicy.Workspace` is declared; the Provider obligations for it are in [Attachment](./workspace-provider.md#attachment). +`Bootstrap.Workspace` and `ResumeRequest.Workspace` carry an external workspace binding exactly when the Provider's `DeploymentPolicy.Workspace` is declared; the Provider obligations for it are in [Attachment](./workspace-provider.md#attachment). Core sets `Bootstrap.SuspendControl` exactly when the Provider declares checkpoint support, and the Provider passes it unchanged in the [Runtime bootstrap](./runtime-bootstrap.md). ### Explicit operation contracts @@ -62,10 +62,10 @@ Every provider returns a complete `ProviderOperations()` declaration with one en | --- | --- | --- | | `Create`, `GetInfo`, `Renew`, `Kill`, `RunCommand` | Supported | Allocation lifecycle and bounded commands | | `Observe` | Explicit decision | Ownership-checked read-only observation of one allocation | -| `Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method; supported only by a `nodes` registration | Exact compute incarnations, capture and restore, retained-source resume and cleanup | +| `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method | Exact compute incarnations, capture and restore, retained-source resume and cleanup | | `RestorePreparation`, `SavePreparation` | The same decision for both | Reuse of a finished Environment preparation | -`Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). +`Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, `RenewCompute` extends the native lease of the exact running incarnation where the Provider has one and otherwise observes exactly like `GetCompute`, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). `SavePreparation(reference, key)` stores what preparation wrote under `/environment` (workspace, packages and initialization, never Runtime state) in the allocation's initial compute after a complete preparation, and `RestorePreparation(reference, key)` puts a stored one back and returns `false` when it holds no valid entry. The `key` is opaque: Core derives it from the Project and everything that reaches preparation, never from the Environment, Session, Harness or Agent, and the Provider never interprets it. The Provider owns storage, scoping to its Runtime image, expiry and eviction. An entry is valid for 24 hours after its save, and a restore must not use an expired one. Core calls `RestorePreparation` after the Runtime connects and, on a hit, sends only the `finalize` operation before completing initialization. On a miss it prepares in full and calls `SavePreparation`; a save error is logged and never fails the Session. A Provider that declares neither always prepares in full. The microsandbox adapter keeps one archive per key in a private directory under the Runtime home. @@ -109,7 +109,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed`, `ErrCommandUnconfirmed` and, for `Resume` only, `ErrCheckpointUnusable` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. -Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. `GetCompute` returns `ErrNotFound` only when the owning node has verified that the exact incarnation no longer exists; an instance with the same name and another native ID is `ErrOwnership`, and an unreachable node or backend is an unknown result. A `ComputeState.Status` of `stopped` or `crashed` means the adapter verified that the exact incarnation’s process no longer exists; its record or disks can remain until `KillCompute` removes them. `paused`, `running` without a Runtime connection, and unknown states do not establish termination. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. +Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. `RenewCompute` never revives stopped compute, and Core calls it instead of `GetCompute` while a running allocation's Runtime is connected, because `Renew` addresses only the initial incarnation. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. `GetCompute` returns `ErrNotFound` only when the owning node has verified that the exact incarnation no longer exists; an instance with the same name and another native ID is `ErrOwnership`, and an unreachable node or backend is an unknown result. A `ComputeState.Status` of `stopped` or `crashed` means the adapter verified that the exact incarnation’s process no longer exists; its record or disks can remain until `KillCompute` removes them. `paused`, `running` without a Runtime connection, and unknown states do not establish termination. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. ### Four distinct readiness facts @@ -144,7 +144,7 @@ A node configuration, `sandbox.NodeConfig`, holds `provider`, `generation`, `ins - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial projection. A declared default size must pass `ValidatePolicy("", Policy)`. -- Checkpoint is admitted only for a `nodes` registration, because the common lifecycle suspends only node allocations; registration rejects a `direct` Provider that declares it. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. +- Checkpoint is admitted for both modes: the common lifecycle suspends, retains, restores and cleans up a `direct` allocation exactly as a node allocation, with no node. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery`, `SelectionDiscovery` and `CredentialVerification` use the shared supported or unsupported declaration with a safe reason. A new requirement field needs an explicit validation update and never inherits an existing decision. Requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -193,7 +193,7 @@ An allocation moves to `cleanup_pending` when its Session is deleted or archived ## Checkpoint lifecycle -A Provider that declares the checkpoint operations (`Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute`) lets Core suspend an idle allocation to a snapshot and restore it on the same or another compatible node. Core records the allocation's `compute_phase`; it is `disabled` for a Provider that does not declare checkpoint support, and the other phases move as follows. Deletion, archive, reset and retention expiry release the allocation from any phase after the cleanup described in [What Core guarantees to a Provider](#what-core-guarantees-to-a-provider). +A Provider that declares the checkpoint operations (`Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute`) lets Core suspend an idle allocation to a snapshot and restore it on the same or another compatible node. Core records the allocation's `compute_phase`; it is `disabled` for a Provider that does not declare checkpoint support, and the other phases move as follows. Deletion, archive, reset and retention expiry release the allocation from any phase after the cleanup described in [What Core guarantees to a Provider](#what-core-guarantees-to-a-provider). ```mermaid stateDiagram-v2 @@ -260,6 +260,8 @@ A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `Artif Recovery uses `Resume.ObserveOnly` for the persisted target and operation. `RestoreAttemptClosed` is an exact operation ID, not a generic absence flag: the adapter may return it only after durably closing an attempt that never entered native execution and fencing every late request for that ID. An adapter may also close a durably admitted attempt when it can prove that its current invocation failed before dispatching native restoration. The microsandbox adapter does this for the fresh request that created the admission, on any failure before it dispatches native restoration. An expired wire deadline or an unclassified failure closes the attempt, and Core then persists a new operation ID. A snapshot that is missing, fails identity or integrity verification, or fails the adapter's qualification for this generation, such as a resource, workspace or runtime image mismatch, is also closed and reported as `ErrCheckpointUnusable`. That outcome is durable: `Resume.ObserveOnly` for the same operation reports it again, and late requests for the operation ID stay fenced. A Provider never returns it once native restoration was dispatched. On `ErrCheckpointUnusable` Core never restores that snapshot again and [ends its retention at once](#cold-replacement-after-checkpoint-retention). Once native restoration has been dispatched, an unknown outcome retains the same target, ownership and original retention deadline; a missing native listing, helper death or timeout does not authorize replay. Such a restore can remain unavailable until explicit deletion or ordinary retention expiry, and pending inputs can reach their public deadline while waiting. User deletion still requires exact native settlement before resources and ownership can be released. +The E2B adapter checkpoints with the SDK's native pause and `connect`. One sandbox spans every generation of an allocation: `Suspend` pauses it with a memory snapshot, so the daemon survives, and `Resume` reconnects to the same sandbox. The helper's private receipt records the current generation, so a superseded generation is `ErrNotFound`, and `KillCompute` and `DeleteSnapshot` never touch a sandbox that another generation owns. `CheckpointCompatibility` names the endpoint and installation and the template build, and a snapshot that does not match is `ErrCheckpointUnusable` before any native call. E2B keeps no independent workspace filesystem, so retention expiry follows the unqualified branch below. + The microsandbox adapter requires an explicit private `checkpoint_root`, outside every guest-accessible filesystem. Its directory is mode `0700`; a private namespace marker identifies the actual checkpoint store. The default installation creates a node-local private store. Cross-node restoration requires operators to mount the same private store on each participating node; mount paths may differ. Each retained checkpoint is an uncompressed native snapshot of the guest's memory and root-disk layers, so a shared store carries that volume for every suspension and cross-node restore. Runtime state directories and workspace bindings are never used to infer this root. The adapter verifies the checkpoint, external filesystem identity and native execution class again before restoration. Restored RAM and file descriptors do not promise continuity of external TCP connections or replay safety for application side effects. ### Cold replacement after checkpoint retention diff --git a/docs/web/console-api-usage.md b/docs/web/console-api-usage.md index 48e577463..f5c5e0329 100644 --- a/docs/web/console-api-usage.md +++ b/docs/web/console-api-usage.md @@ -114,7 +114,7 @@ The list carries each Harness's configuration, so the console does not read `GET | Remove node | `DELETE /core/v1/sandbox/nodes/{node_id}` | Confirmed node removal; the row goes only after Core acknowledges the deletion, and a Clean up the host dialog then gives the host's uninstall command (requiring root or sudo; for a node enrolled with another address than the deployment's, also with `--force`, which skips the installer's confirmation with Core) | | Runtime observations | `GET /core/v1/sandbox/runtime-observations` | Sandbox metrics: hosted Runtimes of every project, each labelled with its project; an E2B sandbox's dialog adds its `observation.disk` as used / limit (null elsewhere) | -An E2B deployment has no nodes; its API key is write-only. Overview and Sandbox metrics count its running and starting sandboxes from the deployment's `resources.allocations` and `resources.pending`, while the hosted Runtime rows come from Runtime observations. The two sources refresh independently, so the console does not infer retention or cleanup from their difference. +An E2B deployment has no nodes; its API key is write-only. Overview and Sandbox metrics count its running and starting sandboxes from the deployment's `resources.running` and `resources.pending`, while the hosted Runtime rows come from Runtime observations. The two sources refresh independently, so the console does not infer retention or cleanup from their difference. ## Writes diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index dcd137d1b..a5cc5f88e 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: 5a6b2fff6d4f54505022e783ebb4a4c20cc8602935842053416870786ee7bc74 +source_hash: 68bb957984c615d36b941efae570b366b6245f08fe4c76561f8aaa92966e1a4b --- Runtime 引导协议连接沙箱提供商与其启动的托管 Runtime。提供商向 Runtime 交付一个引导文件,该文件是此次启动中 Runtime 唯一的认证输入,也是其 Environment 绑定的唯一来源。 @@ -55,11 +55,16 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `environment_id` | Runtime 所服务 Environment 的规范非零 UUID | | `session_id` | Runtime 保存其原生历史的 Session 的规范非零 UUID | | `workspace` | 工作区的干净绝对目录;打包的 Runtime 布局中为 `runtimebootstrap.RuntimeWorkspace`。Runtime 将其作为 `runtimebootstrap.PublicWorkspace`(`/workspace`)提供,这是应用与 Core 使用的路径 | +| `suspend_control` | 必填布尔值。当且仅当提供商声明[检查点支持](./sandbox-provider.md)时,Core 将其置为 true,提供商原样转交 | 解码器拒绝未知、重复、缺失和大小写别名字段,拒绝其他版本及超过 `runtimebootstrap.MaxBytes`的文档。错误不包含提交的值。文件缺失或格式错误时,daemon 在连接前失败。 该文件是此次启动唯一的认证输入:daemon 拒绝将其与自托管注册选项组合使用,并将凭据读入内存而不保存到存储的 profile。凭据不进入命令参数、环境变量或回执。提供商为进程重启保留该文件,仅在明确清理自己拥有的资源时删除。 +## 挂起控制 {#suspend-control} + +`suspend_control` 为 true 时,daemon 在 `runtimebootstrap.SuspendControlFile` 发布其挂起控制并为挂起而停放。Core 的唤醒命令 `oac-daemon resume --control-file` 读取该文件。声明检查点支持的提供商在启动 daemon 前准备该文件所在目录:仅 Runtime 账户可访问(权限 0700,属主 UID 1000)。`suspend_control` 为 true 而 `oac-daemon connect` 无法发布该控制时,连接失败;没有任何其他设置可以启用该控制。 + ## 超时与限制 {#timeouts-and-limits} | 数值 | 适用对象 | 超出时 | diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index abcb0c8cb..aa459bb25 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "沙箱提供商协议" source: docs/sandbox-provider.md -source_hash: 199dc6de344085e9b8c1048e7e8808aa40353878865061236e1a919ad689e28b +source_hash: 0ffc892ad458e104e5e9eba3427429b3f82e7376bfdf3eeac0c9653cee434db1 --- 沙箱提供商协议连接 Core 与为 Core 管理的(`openai_hosted`)Environment 提供计算资源的 adapter,并由该 adapter 运行启动 Runtime daemon 的有界引导流程。本页面向 adapter 作者:说明提供商必须实现什么,以及可以依赖 Core 做什么。Docker、microsandbox 和 E2B 实现了该协议。 @@ -54,7 +54,7 @@ source_hash: 199dc6de344085e9b8c1048e7e8808aa40353878865061236e1a919ad689e28b Docker 等没有原生可续期租约的 backend 仍遵守 Core 的 hosted expiry 和 cleanup 要求。每个 adapter 都实现 `RunCommand`,测试也执行它,但 Core 编排不调用它;只有 node transport 转发。机密命令输入通过 `Command.Stdin` 传输,不放在参数或日志中;结果保留字节顺序、有界输出和实际退出状态。 -`Bootstrap.Workspace` 和 `ResumeRequest.Workspace` 仅当提供商的 `DeploymentPolicy.Workspace` 已声明时才携带外部工作区绑定;提供商对此的义务见[挂载](workspace-provider.md#attachment)。 +`Bootstrap.Workspace` 和 `ResumeRequest.Workspace` 仅当提供商的 `DeploymentPolicy.Workspace` 已声明时才携带外部工作区绑定;提供商对此的义务见[挂载](workspace-provider.md#attachment)。Core 仅在提供商声明支持 checkpoint 时设置 `Bootstrap.SuspendControl`,提供商将其原样放入 [Runtime bootstrap](runtime-bootstrap.md)。 ### 明确的操作契约 {#explicit-operation-contracts} @@ -64,10 +64,10 @@ Docker 等没有原生可续期租约的 backend 仍遵守 Core 的 hosted expir | --- | --- | --- | | `Create`、`GetInfo`、`Renew`、`Kill`、`RunCommand` | 支持 | Allocation 生命周期与有界命令 | | `Observe` | 明确决定 | 检查所有权、只读地观测一个 allocation | -| `Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致;仅 `nodes` 注册可以支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | +| `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致 | 精确计算实例、捕获与恢复、保留源恢复和清理 | | `RestorePreparation`、`SavePreparation` | 两者决定一致 | 复用已完成的 Environment 准备结果 | -`Initial` 和 `NewCompute` 构造 compute reference,不分配资源;`ResumeCompute` 在暂停中止后仅解冻同一驻留实例;`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go))。 +`Initial` 和 `NewCompute` 构造 compute reference,不分配资源;`ResumeCompute` 在暂停中止后仅解冻同一驻留实例;`RenewCompute` 在提供商有原生租约时延长精确运行实例的租约,否则与 `GetCompute` 完全一样只做观察;`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go))。 `SavePreparation(reference, key)` 在准备完整完成后保存该分配初始 compute 中准备写入 `/environment` 的内容(workspace、packages 和 initialization,从不包括 Runtime 状态),`RestorePreparation(reference, key)` 把已保存的内容放回,没有有效条目时返回 `false`。`key` 是不透明的:Core 由 Project 和所有进入准备的内容推导它,从不使用 Environment、Session、Harness 或 Agent,提供商也从不解释它。存储、按 Runtime 镜像划分范围、过期和淘汰由提供商负责。条目在保存后 24 小时内有效,恢复不得使用已过期的条目。Core 在 Runtime 连接后调用 `RestorePreparation`,命中时只发送 `finalize` 操作即完成初始化。未命中时完整准备并调用 `SavePreparation`;保存错误只记录日志,不会使 Session 失败。两者都未声明的提供商始终完整准备。microsandbox 适配器在 Runtime home 下的私有目录中为每个 key 保存一个归档。 @@ -111,7 +111,7 @@ Microsandbox 在返回创建不存在回执前永久关闭初始 Create 准入 `ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound`、`ErrComputeUnconfirmed`、`ErrCommandUnconfirmed`,以及仅用于 `Resume` 的 `ErrCheckpointUnusable` 保持其定义含义。未分类原生或 transport error 表示未知,不授权重试 mutation。Core 不将提供商 diagnostics 读作生命周期事实,也不暴露原生错误文本或凭据;node transport 将错误映射为固定 code,直接 SDK 细节保持私有。 -Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`;原样持久化 operation ID 与提供商 snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试,不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源,不冷启动已停止源。清理针对精确 compute incarnation 和 snapshot,不针对当前同名实例。只有所属节点已验证精确 incarnation 不再存在时,`GetCompute` 才返回 `ErrNotFound`;同名但 native ID 不同的实例为 `ErrOwnership`,节点或提供商不可达属于未知结果。`ComputeState.Status` 为 `stopped` 或 `crashed` 表示 adapter 已验证该精确 incarnation 的进程不再存在;其记录或磁盘可保留到 `KillCompute` 将其移除。`paused`、没有 Runtime 连接的 `running` 和未知状态均不证明进程已终止。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 +Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`;原样持久化 operation ID 与提供商 snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试,不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源,不冷启动已停止源。`RenewCompute` 从不复活已停止的 compute;运行中 allocation 的 Runtime 保持连接时,Core 调用它而不是 `GetCompute`,因为 `Renew` 只寻址初始 incarnation。清理针对精确 compute incarnation 和 snapshot,不针对当前同名实例。只有所属节点已验证精确 incarnation 不再存在时,`GetCompute` 才返回 `ErrNotFound`;同名但 native ID 不同的实例为 `ErrOwnership`,节点或提供商不可达属于未知结果。`ComputeState.Status` 为 `stopped` 或 `crashed` 表示 adapter 已验证该精确 incarnation 的进程不再存在;其记录或磁盘可保留到 `KillCompute` 将其移除。`paused`、没有 Runtime 连接的 `running` 和未知状态均不证明进程已终止。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 ### 四个独立就绪事实 {#four-distinct-readiness-facts} @@ -146,7 +146,7 @@ node 配置 `sandbox.NodeConfig` 包含 `provider`、`generation`、`installatio - `nodes` 注册仅有 `BuildLocal`,`direct` 注册仅有 `BuildDirect`;缺失、混合或未知 mode 被拒绝。 - configuration adapter 与完整 operation 声明都是必需项,因此不完整注册不能发布部分投影。声明的默认大小必须通过 `ValidatePolicy("", Policy)`。 -- Checkpoint 仅准入 `nodes` 注册,因为公共 lifecycle 只暂停 node allocation;声明 checkpoint 的 `direct` 提供商会被注册拒绝。注册不携带 suspension 数值:Core 对每个声明 checkpoint 支持的提供商应用同一个 [suspension policy](#suspension)。 +- Checkpoint 对两种模式都准入:公共 lifecycle 对 `direct` allocation 的暂停、保留、恢复和清理与 node allocation 完全一致,只是没有 node。注册不携带 suspension 数值:Core 对每个声明 checkpoint 支持的提供商应用同一个 [suspension policy](#suspension)。 configuration adapter 必须非 nil,包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定:`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`,`Discovery`、`SelectionDiscovery` 和 `CredentialVerification` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 需要明确更新验证,不继承已有决定。要求凭据不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整,不证明原生 SDK 行为正确;constructor 和 adapter 契约测试仍然适用。 @@ -195,7 +195,7 @@ stateDiagram-v2 ## 检查点生命周期 {#checkpoint-lifecycle} -声明了 checkpoint 操作(`Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute`)的提供商,使 Core 能把空闲 allocation 暂停为快照,并在同一或另一个兼容 node 上恢复。Core 记录 allocation 的 `compute_phase`;未声明 checkpoint 支持的提供商为 `disabled`,其余阶段按下图变化。删除、归档、重置和保留期到期会在完成[Core 对提供商的保证](#what-core-guarantees-to-a-provider)中描述的清理后,从任意阶段释放 allocation。 +声明了 checkpoint 操作(`Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute`)的提供商,使 Core 能把空闲 allocation 暂停为快照,并在同一或另一个兼容 node 上恢复。Core 记录 allocation 的 `compute_phase`;未声明 checkpoint 支持的提供商为 `disabled`,其余阶段按下图变化。删除、归档、重置和保留期到期会在完成[Core 对提供商的保证](#what-core-guarantees-to-a-provider)中描述的清理后,从任意阶段释放 allocation。 ```mermaid stateDiagram-v2 @@ -262,6 +262,8 @@ Core 在初始化完成、没有 root 或 Subagent Turn 处于排队、进行中 恢复用 `Resume.ObserveOnly` 观察已持久化的 target 与 operation。`RestoreAttemptClosed` 是精确 operation ID,不是通用 absence 标志:adapter 只有在持久关闭从未进入原生执行的 attempt,并隔离该 ID 的所有迟到请求后才能返回。如果 adapter 能证明当前调用在派发原生恢复之前失败,也可以关闭已持久 admitted 的 attempt。microsandbox adapter 对创建该 admission 的 fresh 请求,在派发原生恢复之前的任何失败上执行这种关闭。wire deadline 到期或未分类失败会关闭该 attempt,Core 随后持久化新的 operation ID。快照缺失、未通过身份或完整性校验,或未通过 adapter 对当前 generation 的资格检查(例如资源、workspace 或 runtime image 不匹配)时,也会关闭该 attempt,并报告为 `ErrCheckpointUnusable`。该结果是持久的:同一 operation 的 `Resume.ObserveOnly` 会再次报告它,该 operation ID 的迟到请求仍被隔离。一旦已派发原生恢复,提供商不会返回它。收到 `ErrCheckpointUnusable` 后,Core 不再恢复该快照,并[立即结束其保留](#cold-replacement-after-checkpoint-retention)。一旦已派发原生恢复,结果未知的 attempt 保留相同 target、所有权与原始 retention deadline;原生列表缺失、helper 死亡或超时都不授权重放。这种恢复可能一直不可用,直到显式删除或正常保留期到期;等待中的 input 可能达到其公共 deadline。用户删除仍需精确原生 settlement,之后才能释放资源与所有权。 +E2B adapter 使用 SDK 的原生 pause 与 `connect` 做 checkpoint。一个 sandbox 跨越 allocation 的所有 generation:`Suspend` 以内存快照暂停它,daemon 因此存活,`Resume` 重新连接同一个 sandbox。helper 的私有 receipt 记录当前 generation,所以被取代的 generation 返回 `ErrNotFound`,`KillCompute` 和 `DeleteSnapshot` 从不触碰由其他 generation 拥有的 sandbox。`CheckpointCompatibility` 指明 endpoint、安装和模板构建,不匹配的快照在任何原生调用之前即为 `ErrCheckpointUnusable`。E2B 没有独立的 workspace 文件系统,因此保留期到期走下面的未满足条件分支。 + microsandbox adapter 要求显式的私有 `checkpoint_root`,位于所有 guest 可访问文件系统之外。目录权限为 `0700`,私有 namespace marker 标识真实检查点存储。默认安装创建 node 本地私有 store。跨 node 恢复要求运维为参与 node 挂载同一个私有 store,挂载路径可以不同。每个保留的检查点都是 guest 内存与根磁盘层的未压缩原生快照,因此共享 store 要为每次挂起和跨 node 恢复承载这部分数据量。Runtime state 目录与 workspace binding 都不用于推导该 root。adapter 在恢复前再次验证检查点、外部文件系统身份和原生 execution class。恢复 RAM 与文件描述符不保证外部 TCP 连接连续,也不保证应用副作用可安全重放。 ### 检查点保留期后的冷替换 {#cold-replacement-after-checkpoint-retention} diff --git a/docs/zh/web/console-api-usage.md b/docs/zh/web/console-api-usage.md index 607b94c3e..442f58add 100644 --- a/docs/zh/web/console-api-usage.md +++ b/docs/zh/web/console-api-usage.md @@ -1,7 +1,7 @@ --- title: "控制台 API 使用" source: docs/web/console-api-usage.md -source_hash: 7b5d8c6ff67f76a8d005aa2a3e1fdfdd6ea06e7856baf20a6d26e21ea1287475 +source_hash: 4c21f70ece3e6ef7f96cf4b9fda57c14c034196f51c224fcf63174ed0bc88fc1 --- 本页列出各控制台页面调用的 Core 路由,以及控制台如何限定读取范围。本页面向控制台开发者;运维人员请阅读 [Web 控制台](index.md)。[Core 管理 API](../../../contracts/agents-api/zh/admin-api.md) 定义了路由、响应结构、分页和审计记录;[API 命名空间和凭据](../api/index.md) 定义了本文使用的术语。 @@ -116,7 +116,7 @@ source_hash: 7b5d8c6ff67f76a8d005aa2a3e1fdfdd6ea06e7856baf20a6d26e21ea1287475 | 移除节点 | `DELETE /core/v1/sandbox/nodes/{node_id}` | 确认后移除节点;只有 Core 确认删除后该行才会消失,随后 **清理主机** 对话框会提供主机的卸载命令(需要 root 或 sudo;对于使用不同于部署地址的地址注册的节点,还需要使用 `--force`,以跳过安装程序与 Core 的确认) | | Runtime 观测 | `GET /core/v1/sandbox/runtime-observations` | **沙箱监控**:每个项目的托管 Runtime,每项均以所属项目为标签;E2B Sandbox 对话框还会将 `observation.disk` 显示为已用量/限制值(其他位置为 null) | -E2B 部署没有节点;其 API 密钥为只写。**概览** 和 **沙箱监控** 根据部署的 `resources.allocations` 和 `resources.pending` 统计其正在运行和正在启动的 Sandbox,而托管 Runtime 行来自 Runtime 观测。两个来源独立刷新,因此控制台不会根据两者之间的差异推断保留或清理状态。 +E2B 部署没有节点;其 API 密钥为只写。**概览** 和 **沙箱监控** 根据部署的 `resources.running` 和 `resources.pending` 统计其正在运行和正在启动的 Sandbox,而托管 Runtime 行来自 Runtime 观测。两个来源独立刷新,因此控制台不会根据两者之间的差异推断保留或清理状态。 ## 写入 {#writes} diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index 5cbb87ff9..ca3aecf14 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" ) -const Version = 2 +const Version = 3 const MaxBytes = 16 * 1024 // PublicWorkspace is the workspace path applications and Core use for a hosted Environment. @@ -25,20 +25,27 @@ const PublicWorkspace = "/workspace" // RuntimeWorkspace is the workspace directory of the packaged Runtime layout. const RuntimeWorkspace = "/environment/workspace" +// SuspendControlFile is where the Runtime daemon publishes its suspend control. +// Its directory is private to the Runtime user. The Runtime publishes it exactly when +// the bootstrap sets SuspendControl, and Core's wake command reads it. +const SuspendControlFile = "/run/oac/daemon-suspend.json" + var ErrInvalid = errors.New("invalid Runtime bootstrap input") // Connection is a current-version launch input, not Runtime's private auth store. // CoreURL includes the machine API base path. The provider owns delivery and // file permissions; Runtime owns validation and its authentication representation. // EnvironmentID, SessionID and Workspace bind the Runtime to its Environment. +// SuspendControl is true exactly when the Provider declares checkpoint support. type Connection struct { - Version int `json:"version"` - CoreURL string `json:"core_url"` - DeviceID string `json:"device_id"` - Credential string `json:"credential"` - EnvironmentID string `json:"environment_id"` - SessionID string `json:"session_id"` - Workspace string `json:"workspace"` + Version int `json:"version"` + CoreURL string `json:"core_url"` + DeviceID string `json:"device_id"` + Credential string `json:"credential"` + EnvironmentID string `json:"environment_id"` + SessionID string `json:"session_id"` + Workspace string `json:"workspace"` + SuspendControl bool `json:"suspend_control"` } func canonicalUUID(value string) bool { @@ -106,6 +113,14 @@ func Decode(raw []byte) (Connection, error) { err = d.Decode(&c.SessionID) case "workspace": err = d.Decode(&c.Workspace) + case "suspend_control": + var enabled *bool + if err = d.Decode(&enabled); err == nil && enabled == nil { + err = ErrInvalid + } + if enabled != nil { + c.SuspendControl = *enabled + } default: return Connection{}, ErrInvalid } @@ -116,7 +131,7 @@ func Decode(raw []byte) (Connection, error) { if token, err = d.Token(); err != nil || token != json.Delim('}') { return Connection{}, ErrInvalid } - if len(seen) != 7 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { + if len(seen) != 8 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { return Connection{}, ErrInvalid } return c, nil diff --git a/internal/runtimebootstrap/bootstrap_test.go b/internal/runtimebootstrap/bootstrap_test.go index 524e8973b..86fe22190 100644 --- a/internal/runtimebootstrap/bootstrap_test.go +++ b/internal/runtimebootstrap/bootstrap_test.go @@ -8,7 +8,7 @@ import ( func TestConnectionRoundTrip(t *testing.T) { input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential", - EnvironmentID: "0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11", SessionID: "6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e", Workspace: RuntimeWorkspace} + EnvironmentID: "0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11", SessionID: "6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e", Workspace: RuntimeWorkspace, SuspendControl: true} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -20,11 +20,14 @@ func TestConnectionRoundTrip(t *testing.T) { } func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { - good := `{"version":2,"core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret","environment_id":"0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11","session_id":"6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e","workspace":"/environment/workspace"}` + good := `{"version":3,"core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret","environment_id":"0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11","session_id":"6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e","workspace":"/environment/workspace","suspend_control":false}` + if _, err := Decode([]byte(good)); err != nil { + t.Fatal("valid input rejected", err) + } for name, raw := range map[string]string{ "null": "null", "array": "[]", "trailing": good + "{}", - "duplicate": strings.Replace(good, `"version":2`, `"version":2,"version":2`, 1), - "unknown": strings.Replace(good, `"version":2`, `"extra":1,"version":2`, 1), + "duplicate": strings.Replace(good, `"version":3`, `"version":3,"version":3`, 1), + "unknown": strings.Replace(good, `"version":3`, `"extra":1,"version":3`, 1), "case": strings.Replace(good, "credential", "Credential", 1), "old auth": `{"server_url":"https://core.example","runtime_id":"old","runner_credential":"test-secret"}`, "oversized": strings.Repeat(" ", MaxBytes) + good, @@ -36,13 +39,14 @@ func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { }) } for field, values := range map[string][]any{ - "version": {nil, 0, 1, "2"}, - "core_url": {"http://user:pass@core.example/api/v1", "https://core.example/api/v1?", "https://core.example/api/v1#", "https://core.example", "https://core.example/api/v1/", "https://core.example/api/v1?q=1", ""}, - "device_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, - "credential": {nil, "", "test\nsecret", "test\x00secret"}, - "environment_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, - "session_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, - "workspace": {"", "workspace", "/", "/environment/../workspace", "/environment/workspace/"}, + "version": {nil, 0, 2, "3"}, + "suspend_control": {nil, 1, "true"}, + "core_url": {"http://user:pass@core.example/api/v1", "https://core.example/api/v1?", "https://core.example/api/v1#", "https://core.example", "https://core.example/api/v1/", "https://core.example/api/v1?q=1", ""}, + "device_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "credential": {nil, "", "test\nsecret", "test\x00secret"}, + "environment_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "session_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "workspace": {"", "workspace", "/", "/environment/../workspace", "/environment/workspace/"}, } { for _, value := range values { var input map[string]any diff --git a/packages/agents-client/src/generated/core-api.ts b/packages/agents-client/src/generated/core-api.ts index b8eeb6163..074141c2f 100644 --- a/packages/agents-client/src/generated/core-api.ts +++ b/packages/agents-client/src/generated/core-api.ts @@ -209,8 +209,9 @@ export type DeploymentMode = (typeof deploymentModeValues)[number]; export interface DeploymentResources { allocations: number; pending: number; + running: number; } -export const deploymentResourcesFields = ["allocations", "pending"] as const; +export const deploymentResourcesFields = ["allocations", "pending", "running"] as const; export interface DeploymentSpec { resources: SandboxResources; workspace?: Declaration; diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index 126533cf9..b944851d4 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -44,11 +44,11 @@ const allocation = { state: "running", compute_phase: "running", compute_phase_changed_at: null, diagnostic: "", initialization: "ready", created_at: created, }; const unconfigured = { - credential_configured: false, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; + credential_configured: false, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, running: 0, pending: 0 }, suspension: null }; const docker = { configuration: {}, metadata: {}, ...unconfigured, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, owner_epoch: 1, generation: 1, mode: "nodes", - resources: { allocations: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, specification_digest: "0".repeat(64), + resources: { allocations: 2, running: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, specification_digest: "0".repeat(64), }; const microsandbox = { ...docker, provider: "microsandbox", specification: { resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 8192 } }, @@ -254,7 +254,7 @@ describe("hosted provider configuration", () => { const e2b = { api_key: "test-only-secret", template: "runtime:00000000-0000-0000-0000-000000000001" }; it("writes E2B configuration and generation without beta headers or browser credentials", async () => { const deployment = { - ...e2bDeployment, generation: 2, resources: { allocations: 0, pending: 0 }, + ...e2bDeployment, generation: 2, resources: { allocations: 0, running: 0, pending: 0 }, configuration: { template: e2b.template, api_url: "https://api.e2b.app", domain: "e2b.app" } , credential_configured: true, metadata: { template_build: { status: "ready", resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 24063 } } }, }; const fetch = vi.fn().mockImplementation(async () => response(deployment)); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 2af93b40f..578760864 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -160,7 +160,7 @@ function projectDeployment(value: unknown): SandboxDeployment { const suspension = deployment.suspension === null ? null : members(deployment.suspension, suspensionFields); const configured = hasOwn(deployment, "specification"); valid(strings(deployment, ["installation_id", "core_url"]) && (deployment.provider === "" || (typeof deployment.provider === "string" && hasOwn(deploymentContract.providers, deployment.provider))) && isOneOf(deploymentModeValues, deployment.mode) && - [deployment.owner_epoch, deployment.generation, resources.allocations, resources.pending].every(isNonnegativeInteger) && + [deployment.owner_epoch, deployment.generation, resources.allocations, resources.running, resources.pending].every(isNonnegativeInteger) && (suspension === null || [suspension.idle_seconds, suspension.retention_seconds].every(isNonnegativeInteger)) && configured === hasOwn(deployment, "specification_digest") && (!configured || (typeof deployment.specification_digest === "string" && deployment.specification_digest !== ""))); return { diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index bacf142d9..085942b2a 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -105,6 +105,17 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc defer done() return v.ResumeCompute(ctx, r, c) } +func (p *generationRouter) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if err := providercontract.Require(p, "RenewCompute"); err != nil { + return sandbox.ComputeState{}, err + } + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.ComputeState{}, err + } + defer done() + return v.RenewCompute(ctx, r, c) +} func (p *generationRouter) RestorePreparation(ctx context.Context, r sandbox.Reference, key string) (bool, error) { if err := providercontract.Require(p, "RestorePreparation"); err != nil { return false, err diff --git a/services/core/deploy/e2b/README.md b/services/core/deploy/e2b/README.md index 93a7d6c55..bb244e9f7 100644 --- a/services/core/deploy/e2b/README.md +++ b/services/core/deploy/e2b/README.md @@ -21,7 +21,7 @@ python -m venv "$HOME/.oac/build/e2b-sdk" --output "$HOME/.oac/build/e2b-template.json" ``` -[`build-template.py`](build-template.py) requires a `sha256:` image ID, a Linux amd64 image and the Runtime layout. It copies the image's `/usr/local/bin`, `/opt` and, when present, `/usr/local/codex-resources` and `/etc/codex`, together with its `HOME` and `OAC_*` environment, onto a digest-pinned `node:22.23.1-bookworm-slim` base with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip`, `ripgrep` and `util-linux`. It installs [`init.py`](init.py) and [`managed_init.py`](managed_init.py) read-only under `/opt/oac-e2b`, runs the daemon as UID/GID 1000 (`runtime`, home `/home/runtime`) and builds with 2 vCPUs and 2048 MiB. Only the `usr`, `usr/local` and `etc` archive ancestors it creates get traversable modes; Runtime file modes, private build contexts, key inputs and the output's umask stay unchanged. +[`build-template.py`](build-template.py) requires a `sha256:` image ID, a Linux amd64 image and the Runtime layout. It copies the image's `/usr/local/bin`, `/opt` and, when present, `/usr/local/codex-resources` and `/etc/codex`, together with its `HOME` and `OAC_*` environment, onto a digest-pinned `node:22.23.1-bookworm-slim` base with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip`, `ripgrep` and `util-linux`. Rebuild a template whenever the Runtime bootstrap version changes: Create refuses a template whose contract names another version. It installs [`init.py`](init.py) and [`managed_init.py`](managed_init.py) read-only under `/opt/oac-e2b`, runs the daemon as UID/GID 1000 (`runtime`, home `/home/runtime`) and builds with 2 vCPUs and 2048 MiB. Only the `usr`, `usr/local` and `etc` archive ancestors it creates get traversable modes; Runtime file modes, private build contexts, key inputs and the output's umask stay unchanged. The output file records `template` (the immutable `templateID:build_UUID`), the source `image`, the packaged `runtime_sha256` and the `base`. Use that exact `template` value. The template must qualify every Harness its image advertises. Install system dependencies into the image; the daemon runs as UID/GID 1000. No E2B account key, executor key or model credential belongs in a build, template environment, metadata, command argument or log. diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index ee81a1327..bee1ee648 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,6 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" -ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] +ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] @@ -9,9 +9,11 @@ MAX_OUTPUT = 1048576 MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute","renew_compute","suspend","resume","kill_compute","delete_snapshot","command_compute","resume_compute"] PROTOCOL_VERSION = 1 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Snapshot","Deadline"] +RESPONSE_FIELDS = ["Version","Info","Command","State","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RUNTIME_BOOTSTRAP_VERSION = 3 SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index 3cd2e11cc..a4e6619bf 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -44,6 +44,11 @@ def initialize(): binding = identity(payload) shared.write_private(root / 'managed-launch.json', binding) environment = shared.prepare_runtime() + # The daemon publishes its suspend control here; Core's wake command reads it. + control = Path(contract.SUSPEND_CONTROL_FILE) + control.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chown(control.parent, 1000, 1000) + control.parent.chmod(0o700) connection = Path(environment['OAC_RUNTIME_HOME']).parent / 'runtime-bootstrap.json' shared.write_private(connection, payload['RuntimeBootstrap'], owner=1000) source.unlink() diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index ae6118a2c..fdfc446a9 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -16,9 +16,9 @@ def payload(): value = {key: str(uuid4()) for key in ['InstallationID', 'TenantID', 'EnvironmentID', 'AllocationID', 'SessionID', 'DeviceID']} - return dict(value, RuntimeBootstrap={'version': 2, 'core_url': 'https://core.example/api/v1', + return dict(value, RuntimeBootstrap={'version': 3, 'core_url': 'https://core.example/api/v1', 'device_id': value['DeviceID'], 'credential': 'private-managed-token', - 'environment_id': value['EnvironmentID'], 'session_id': value['SessionID'], 'workspace': '/environment/workspace'}) + 'environment_id': value['EnvironmentID'], 'session_id': value['SessionID'], 'workspace': '/environment/workspace', 'suspend_control': True}) class ManagedStartupTest(unittest.TestCase): @@ -63,7 +63,9 @@ def exercise(self, failed=False): image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'OAC_RUNTIME_HOME': str(Path(temporary) / '.oac')} with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ - patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): + patch.object(managed_init.os, 'fchown'), patch.object(managed_init.os, 'chown'), \ + patch.object(managed_init.contract, 'SUSPEND_CONTROL_FILE', str(Path(temporary) / 'run' / 'daemon-suspend.json')), \ + patch.object(managed_init.subprocess, 'Popen', process): if failed: with self.assertRaises(RuntimeError): managed_init.initialize() @@ -80,6 +82,7 @@ def exercise(self, failed=False): self.assertNotIn(data['RuntimeBootstrap']['credential'], json.dumps(process.call_args.kwargs['env'])) self.assertEqual(json.loads(connection_file.read_text())['environment_id'], data['EnvironmentID']) self.assertEqual(process.call_args.kwargs['user'], 1000) + self.assertTrue((Path(temporary) / 'run').is_dir()) if failed: self.assertFalse((root / 'managed-ready.json').exists()) else: diff --git a/services/core/internal/db/queries/sandbox_reset.sql b/services/core/internal/db/queries/sandbox_reset.sql index 335321ac4..cbadae68d 100644 --- a/services/core/internal/db/queries/sandbox_reset.sql +++ b/services/core/internal/db/queries/sandbox_reset.sql @@ -100,6 +100,7 @@ held AS ( SELECT sqlc.embed(d), (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, + (SELECT count(*) FROM runtime_allocations a WHERE a.state = 'running' AND a.compute_phase IN ('running', 'disabled'))::bigint AS running, jsonb_build_object( 'busy', (SELECT count(*) FROM classified WHERE category = 'busy'), 'idle', (SELECT count(*) FROM classified WHERE category = 'idle'), diff --git a/services/core/internal/db/sqlc/sandbox_reset.sql.go b/services/core/internal/db/sqlc/sandbox_reset.sql.go index 302fe1620..4412e8214 100644 --- a/services/core/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/core/internal/db/sqlc/sandbox_reset.sql.go @@ -93,6 +93,7 @@ held AS ( SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.updated_at, d.provider_kind, d.owner_epoch, d.generation, d.mode, d.provider_credential, d.specification, d.reset_clear, d.reset_requested_at, d.reset_deadline_at, d.reset_forced_at, d.reset_audit, d.provider_config, d.provider_metadata, d.idle_seconds, (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, + (SELECT count(*) FROM runtime_allocations a WHERE a.state = 'running' AND a.compute_phase IN ('running', 'disabled'))::bigint AS running, jsonb_build_object( 'busy', (SELECT count(*) FROM classified WHERE category = 'busy'), 'idle', (SELECT count(*) FROM classified WHERE category = 'idle'), @@ -117,6 +118,7 @@ type GetSandboxDeploymentSnapshotRow struct { RuntimeDeployment RuntimeDeployment `json:"runtime_deployment"` Allocations int64 `json:"allocations"` Pending int64 `json:"pending"` + Running int64 `json:"running"` Remaining []byte `json:"remaining"` Rollout []byte `json:"rollout"` } @@ -145,6 +147,7 @@ func (q *Queries) GetSandboxDeploymentSnapshot(ctx context.Context) (GetSandboxD &i.RuntimeDeployment.IdleSeconds, &i.Allocations, &i.Pending, + &i.Running, &i.Remaining, &i.Rollout, ) diff --git a/services/core/internal/deployment/nodes.go b/services/core/internal/deployment/nodes.go index 1ad434a86..2b0ab60cd 100644 --- a/services/core/internal/deployment/nodes.go +++ b/services/core/internal/deployment/nodes.go @@ -13,7 +13,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) @@ -605,7 +604,7 @@ func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses if err != nil { return err } - checkpointSupported := adapter.Operations()["Initial"].State == providercontract.Supported + checkpointSupported := sandbox.SupportsCheckpoint(adapter.Operations()) seen := map[uint64]bool{} for _, status := range statuses { if status.Checkpoint != nil && (status.State != "ready" || !checkpointSupported || status.Checkpoint.Validate() != nil) || status.State == "ready" && checkpointSupported && status.Checkpoint == nil { diff --git a/services/core/internal/deployment/view.go b/services/core/internal/deployment/view.go index 88cf94ef7..253fa0da0 100644 --- a/services/core/internal/deployment/view.go +++ b/services/core/internal/deployment/view.go @@ -44,9 +44,11 @@ type View struct { CoreURL string `json:"core_url" binding:"required"` } -// Resources counts what still belongs to the deployment. +// Resources counts what still belongs to the deployment. Running excludes +// suspended allocations, which Allocations includes. type Resources struct { Allocations int64 `json:"allocations" binding:"required"` + Running int64 `json:"running" binding:"required"` Pending int64 `json:"pending" binding:"required"` } diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 58b17c3a8..44acb244f 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -24,6 +24,7 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -56,6 +57,9 @@ func (*lifecycleOnlySandbox) RunCommandCompute(context.Context, sandbox.Referenc func (*lifecycleOnlySandbox) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: "fixture_operation_not_supported"} } +func (*lifecycleOnlySandbox) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} func (*lifecycleOnlySandbox) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index f3bc4f863..4c309e6df 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -60,8 +60,8 @@ func logComputeStep(ctx context.Context, owner deployment.Allocation, step strin func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.Allocation) (err error) { defer func() { err = withObservationOwner(owner, err) }() p := r.config.Provider - if err := providercontract.Require(p, "Initial"); err != nil { - return err + if !sandbox.SupportsCheckpoint(p.ProviderOperations()) { + return providercontract.Require(p, "Initial") } initial, err := p.Initial(ctx, runtimeReference(owner)) if err != nil { @@ -81,8 +81,8 @@ func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.A func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment.Allocation) (err error) { defer func() { err = withObservationOwner(owner, err) }() p := r.config.Provider - if err := providercontract.Require(p, "Initial"); err != nil { - return err + if !sandbox.SupportsCheckpoint(p.ProviderOperations()) { + return providercontract.Require(p, "Initial") } var state runtimeCompute if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { @@ -121,7 +121,14 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) (err error) { defer func() { err = withObservationOwner(owner, err) }() - compute, err := p.GetCompute(ctx, runtimeReference(owner), state.Current) + // With an open Runtime connection the observation also extends the native + // lease, as the disabled path does; otherwise it only observes. + observe := p.GetCompute + if r.runtimeConnected(owner.DeviceID) { + observe = p.RenewCompute + } + peer, peerErr := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) + compute, err := observe(ctx, runtimeReference(owner), state.Current) if errors.Is(err, sandbox.ErrNotFound) { return r.releaseLostCompute(ctx, p, owner, state, err) } @@ -137,9 +144,8 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro if compute.Status != "running" || !compute.BootstrapComplete { return sandbox.ErrComputeUnconfirmed } - peer, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) - if err != nil { - return err + if peerErr != nil { + return peerErr } if _, err := r.deployment.CheckRunning(ctx, owner); err != nil { return err diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index a3578a837..125d5ede8 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -6,6 +6,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -31,7 +32,7 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro // This idempotent control signal is fenced by guest PID/start time and the // suspension token. It cannot execute or replay an agent request. started := time.Now() - result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", "/run/oac/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) + result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", runtimebootstrap.SuspendControlFile, "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) logComputeStep(ctx, owner, "wake_command", started, err) if err != nil { return err diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index ff8de1e0e..2265f6484 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -109,12 +109,9 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. if copied.ProviderKind == "" || (copied.Mode != string(sandbox.DeploymentNodes) && copied.Mode != string(sandbox.DeploymentDirect)) { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == string(sandbox.DeploymentDirect) && copied.Suspension != nil { - return RuntimeProvider{}, sandbox.ErrInvalid - } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.Retention < time.Second { + if !sandbox.SupportsCheckpoint(config.Provider.ProviderOperations()) || policy.Retention < time.Second { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy @@ -280,6 +277,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, + SuspendControl: sandbox.SupportsCheckpoint(provider.ProviderOperations()), }) logComputeStep(ctx, owner, "create", started, err) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { @@ -469,7 +467,7 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner deployment.Allocat if r.config.Suspension != nil && environment.Initialization == "complete" { return r.enableCompute(ctx, owner) } - if peer, err := r.registry.LookupDevice(owner.DeviceID); err != nil || peer.IsClosed() { + if !r.runtimeConnected(owner.DeviceID) { return nil } renewed, err := provider.Renew(ctx, runtimeReference(owner)) @@ -483,6 +481,13 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner deployment.Allocat return err } +// runtimeConnected is the renewal condition: running compute keeps its native +// lease while the allocation's Runtime holds an open connection. +func (r *runtimeLifecycle) runtimeConnected(device string) bool { + peer, err := r.registry.LookupDevice(device) + return err == nil && !peer.IsClosed() +} + // Environment identity owns connectivity; preparation has an independent owner. func (r *runtimeLifecycle) clearRuntimeState(owner deployment.Allocation) { delete(r.connections, owner.EnvironmentID) diff --git a/services/core/internal/execution/runtime_lost_compute_test.go b/services/core/internal/execution/runtime_lost_compute_test.go index 365c47e27..d30cf62e1 100644 --- a/services/core/internal/execution/runtime_lost_compute_test.go +++ b/services/core/internal/execution/runtime_lost_compute_test.go @@ -33,6 +33,9 @@ type lostComputeProvider struct { func (p *lostComputeProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { return p.observe(c) } +func (p *lostComputeProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, r, c) +} func (p *lostComputeProvider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { p.killed = append(p.killed, c) if p.onKill != nil { diff --git a/services/core/internal/execution/runtime_pending_test.go b/services/core/internal/execution/runtime_pending_test.go index 575d32667..0d8063e2c 100644 --- a/services/core/internal/execution/runtime_pending_test.go +++ b/services/core/internal/execution/runtime_pending_test.go @@ -3,6 +3,7 @@ package execution import ( "context" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "net/http" "net/http/httptest" "net/url" @@ -381,3 +382,19 @@ func TestAwaitingConnectionPassSharesCancellationFence(t *testing.T) { t.Fatal("connection pass escaped cancellation") } } + +// noCheckpoint declares a fixture Provider that supports no checkpoint. +func noCheckpoint() providercontract.Operations { + return providercontract.Operations{"Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}} +} + +func (*connectedBootstrapProvider) ProviderOperations() providercontract.Operations { + return noCheckpoint() +} +func (*settledWorkspaceRefusal) ProviderOperations() providercontract.Operations { + return noCheckpoint() +} +func (*unknownWorkspaceCreation) ProviderOperations() providercontract.Operations { + return noCheckpoint() +} +func (*preparationProvider) ProviderOperations() providercontract.Operations { return noCheckpoint() } diff --git a/services/core/internal/persistence/postgres/deploymentpg/records.go b/services/core/internal/persistence/postgres/deploymentpg/records.go index a17f2a5a9..ea89846a9 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/records.go +++ b/services/core/internal/persistence/postgres/deploymentpg/records.go @@ -75,7 +75,7 @@ func record(d sqlc.RuntimeDeployment, cipher *credentialcrypto.Cipher, open bool } func snapshot(row sqlc.GetSandboxDeploymentSnapshotRow) (deployment.Snapshot, error) { - result := deployment.Snapshot{Record: record(row.RuntimeDeployment, nil, false), Resources: deployment.Resources{Allocations: row.Allocations, Pending: row.Pending}} + result := deployment.Snapshot{Record: record(row.RuntimeDeployment, nil, false), Resources: deployment.Resources{Allocations: row.Allocations, Running: row.Running, Pending: row.Pending}} if err := json.Unmarshal(row.Rollout, &result.Rollout); err != nil { return deployment.Snapshot{}, err } diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index 0e2ac9c0d..784193167 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -23,6 +23,7 @@ func Operations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "docker_does_not_support_preparation_cache"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "docker_does_not_support_preparation_cache"}, "Observe": {State: providercontract.Supported}, @@ -56,6 +57,9 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} func (p *Provider) RestorePreparation(context.Context, sandbox.Reference, string) (bool, error) { return false, &providercontract.UnsupportedError{Operation: "RestorePreparation", Reason: Operations()["RestorePreparation"].Reason} } diff --git a/services/core/internal/sandbox/e2b/checkpoint.go b/services/core/internal/sandbox/e2b/checkpoint.go new file mode 100644 index 000000000..4c5242cfc --- /dev/null +++ b/services/core/internal/sandbox/e2b/checkpoint.go @@ -0,0 +1,180 @@ +package e2b + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// One E2B sandbox spans every generation of an allocation: pause keeps the +// sandbox and resume reconnects to it. The helper's private receipt records the +// current generation, so only that generation addresses the sandbox. + +func digest(parts ...string) string { + sum := sha256.Sum256([]byte(strings.Join(parts, "\n"))) + return hex.EncodeToString(sum[:]) +} + +// compatibility names the endpoint and installation that own a paused sandbox +// and the template build it runs; a snapshot restores only under the same two. +func (c Config) compatibility() sandbox.CheckpointCompatibility { + apiURL, domain, _ := NormalizeEndpoint(c.APIURL, c.Domain) + return sandbox.CheckpointCompatibility{ArtifactDomain: digest("e2b-endpoint", c.InstallationID, apiURL, domain), ExecutionClass: digest("e2b-template", c.Template)} +} + +func (p *Provider) name(r sandbox.Reference, generation uint64) string { + sum := sha256.Sum256([]byte(p.config.InstallationID + ":" + r.TenantID + ":" + r.EnvironmentID + ":" + r.AllocationID)) + return fmt.Sprintf("oac-%x-g%d", sum[:16], generation) +} + +func (p *Provider) validSnapshot(r sandbox.Reference, s sandbox.SnapshotIdentity) bool { + return s.Compatibility.Validate() == nil && validID(s.OperationID) && s.SourceName == p.name(r, s.SourceGeneration) && + s.Reference == s.SourceName+":s-"+s.OperationID && s.SourceID != "" && s.ID != "" +} + +func (p *Provider) validCompute(r sandbox.Reference, c sandbox.Compute) bool { + if !validReference(r) || c.Name != p.name(r, c.Generation) { + return false + } + if c.Generation == 0 { + return c.RestoredFrom == nil + } + return c.RestoredFrom != nil && p.validSnapshot(r, *c.RestoredFrom) && c.RestoredFrom.SourceGeneration < c.Generation +} + +func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Compute, error) { + if err := ctx.Err(); err != nil { + return sandbox.Compute{}, err + } + if !validReference(r) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return sandbox.Compute{Name: p.name(r, 0)}, nil +} + +func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { + if err := ctx.Err(); err != nil { + return sandbox.Compute{}, err + } + c := sandbox.Compute{Generation: generation, Name: p.name(r, generation)} + if snapshot != nil { + value := *snapshot + c.RestoredFrom = &value + } + if !p.validCompute(r, c) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return c, nil +} + +// state sends one checkpoint request and validates the returned incarnation +// against the one the request named. +func (p *Provider) state(ctx context.Context, q Request, want sandbox.Compute) (sandbox.ComputeState, error) { + deadline, ok := ctx.Deadline() + if !ok || !p.validCompute(q.Reference, want) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + if err := ctx.Err(); err != nil { + return sandbox.ComputeState{}, err + } + q.Version, q.Config, q.Deadline = ProtocolVersion, p.config, deadline + out, err := p.exchange(ctx, q) + if err != nil { + return sandbox.ComputeState{}, err + } + s := out.State + if s == nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + if s.RestoreAttemptClosed != "" { + if q.Resume == nil || !s.ClosesRestoreAttempt(*q.Resume) { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *s, nil + } + got := s.Compute + if got.ID == "" || got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) || + (got.RestoredFrom == nil) != (want.RestoredFrom == nil) || want.RestoredFrom != nil && *got.RestoredFrom != *want.RestoredFrom { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + if q.Suspend != nil { + snapshot := s.Snapshot + if snapshot == nil { + if !q.Suspend.ObserveOnly || s.SourceStopped || !s.BootstrapComplete || s.Status != "running" && s.Status != "paused" { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + } else if !p.validSnapshot(q.Reference, *snapshot) || snapshot.Compatibility != p.config.Compatibility || snapshot.OperationID != q.Suspend.OperationID || + snapshot.SourceID != want.ID && want.ID != "" || snapshot.SourceID != got.ID || !q.Suspend.ObserveOnly && (!s.SourceStopped || s.Status != "paused") { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + } + return *s, nil +} + +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, Request{Operation: "compute", Reference: r, Compute: &c}, c) +} + +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, Request{Operation: "renew_compute", Reference: r, Compute: &c}, c) +} + +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: &c}, c) +} + +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + if !validID(q.OperationID) || q.Snapshot != nil && !p.validSnapshot(q.Reference, *q.Snapshot) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}, q.Source) +} + +// Resume reconnects to the paused sandbox. A snapshot from another endpoint or +// template build is unusable before any native call. +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + if q.Workspace != nil { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + if !validID(q.OperationID) || !p.validSnapshot(q.Reference, q.Snapshot) || q.Target.RestoredFrom == nil || *q.Target.RestoredFrom != q.Snapshot { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + if q.Snapshot.Compatibility != p.config.Compatibility { + return sandbox.ComputeState{}, sandbox.ErrCheckpointUnusable + } + return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}, q.Target) +} + +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + deadline, ok := ctx.Deadline() + if !ok || !p.validCompute(r, c) { + return sandbox.ErrInvalid + } + _, err := p.exchange(ctx, Request{Version: ProtocolVersion, Operation: "kill_compute", Config: p.config, Reference: r, Compute: &c, Deadline: deadline}) + return err +} + +func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s sandbox.SnapshotIdentity) error { + deadline, ok := ctx.Deadline() + if !ok || !validReference(r) || !p.validSnapshot(r, s) { + return sandbox.ErrInvalid + } + _, err := p.exchange(ctx, Request{Version: ProtocolVersion, Operation: "delete_snapshot", Config: p.config, Reference: r, Snapshot: &s, Deadline: deadline}) + return err +} + +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { + if err := validCommand(command); err != nil { + return sandbox.CommandResult{}, err + } + deadline, ok := ctx.Deadline() + if !ok || !p.validCompute(r, c) { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + out, err := p.exchange(ctx, Request{Version: ProtocolVersion, Operation: "command_compute", Config: p.config, Reference: r, Compute: &c, Command: &command, Deadline: deadline}) + return commandResult(out, err) +} diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index f4cf30ffb..7beb4c8c0 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -20,12 +20,12 @@ const MaxCommandInputBytes = sandbox.MaxCommandInputBytes // HelperOperations declares the complete set of one-shot helper operations. func HelperOperations() []string { - return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential"} + return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential", "compute", "renew_compute", "suspend", "resume", "kill_compute", "delete_snapshot", "command_compute", "resume_compute"} } // HelperErrors are sanitized wire outcomes; an empty code denotes success. func HelperErrors() []string { - return []string{"", "invalid", "ownership", "exists", "not_found", "command_unconfirmed", "unconfirmed", "template_invalid", "team_mismatch", "unauthorized"} + return []string{"", "invalid", "ownership", "exists", "not_found", "command_unconfirmed", "unconfirmed", "template_invalid", "team_mismatch", "unauthorized", "checkpoint_unusable"} } // Validate checks the operation envelope before the helper can start. Native @@ -55,6 +55,11 @@ func (q Request) Validate() error { if !validReference(q.Reference) { return sandbox.ErrInvalid } + present := map[string]bool{"compute": q.Compute != nil, "renew_compute": q.Compute != nil, "kill_compute": q.Compute != nil, "resume_compute": q.Compute != nil, + "command_compute": q.Compute != nil && q.Command != nil, "suspend": q.Suspend != nil, "resume": q.Resume != nil, "delete_snapshot": q.Snapshot != nil} + if want, checkpoint := present[q.Operation]; checkpoint && !want { + return sandbox.ErrInvalid + } } return nil } @@ -69,12 +74,17 @@ type Request struct { Bootstrap *sandbox.Bootstrap `json:",omitempty"` RuntimeBootstrap *runtimebootstrap.Connection `json:",omitempty"` Command *sandbox.Command `json:",omitempty"` + Compute *sandbox.Compute `json:",omitempty"` + Suspend *sandbox.SuspendRequest `json:",omitempty"` + Resume *sandbox.ResumeRequest `json:",omitempty"` + Snapshot *sandbox.SnapshotIdentity `json:",omitempty"` Deadline time.Time } type Response struct { Version int Info *sandbox.Info `json:",omitempty"` Command *sandbox.CommandResult `json:",omitempty"` + State *sandbox.ComputeState `json:",omitempty"` ErrorCode string DeploymentValid bool `json:",omitempty"` TemplateBuild *TemplateBuild `json:",omitempty"` diff --git a/services/core/internal/sandbox/e2b/helper_contract_test.go b/services/core/internal/sandbox/e2b/helper_contract_test.go index a40130f89..76c2dcd18 100644 --- a/services/core/internal/sandbox/e2b/helper_contract_test.go +++ b/services/core/internal/sandbox/e2b/helper_contract_test.go @@ -66,6 +66,7 @@ func validManagedExchange(data []byte) bool { _ = json.Unmarshal(bootstrapBytes, &expected) delete(expected, "CoreURL") delete(expected, "Credential") + delete(expected, "SuspendControl") expected["InstallationID"] = nil expected["RuntimeBootstrap"] = nil if workspace, present := fields["Workspace"]; present { diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 79d48cdd2..aaa39d639 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -67,9 +67,9 @@ func main() { write("services/core/internal/sandbox/e2b/helper_sdk_generated.go", goCode) bootstrap := fields(reflect.TypeFor[sandbox.Bootstrap]()) requiredBootstrap := fieldNames(reflect.TypeFor[sandbox.Bootstrap](), true) - requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "SuspendControl" }) requiredBootstrap = append(requiredBootstrap, "InstallationID", "RuntimeBootstrap") - bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "SuspendControl" }) bootstrap = append(bootstrap, "InstallationID", "RuntimeBootstrap") identity := fields(reflect.TypeFor[sandbox.Reference]()) for _, name := range []string{"SessionID", "DeviceID"} { @@ -81,7 +81,7 @@ func main() { identity = append(identity, "InstallationID") values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, - "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, + "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, "RUNTIME_BOOTSTRAP_VERSION": runtimebootstrap.Version, "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), @@ -131,6 +131,21 @@ func fixtures() []byte { if operation == "verify_credential" { copy.References = []sandbox.Reference{r} } + compute := sandbox.Compute{Name: "fixture", ID: "fixture-id"} + snapshot := sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "a", ExecutionClass: "e"}, Reference: "fixture:s-op", ID: "fixture-id", OperationID: "op", SourceName: "fixture", SourceID: "fixture-id"} + switch operation { + case "compute", "renew_compute", "resume_compute", "kill_compute": + copy.Compute = &compute + case "command_compute": + copy.Compute, copy.Command = &compute, &sandbox.Command{Args: []string{"true"}} + case "suspend": + copy.Suspend = &sandbox.SuspendRequest{Reference: r, OperationID: "op", Source: compute} + case "resume": + target := sandbox.Compute{Generation: 1, Name: "fixture-g1", RestoredFrom: &snapshot} + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: "op", Snapshot: snapshot, Target: target} + case "delete_snapshot": + copy.Snapshot = &snapshot + } add("request", operation, true, copy) } for _, item := range []struct { @@ -182,6 +197,7 @@ func fixtures() []byte { managed := object(b) delete(managed, "CoreURL") delete(managed, "Credential") + delete(managed, "SuspendControl") managed["InstallationID"] = installation managed["RuntimeBootstrap"] = runtimebootstrap.Connection(connection) add("managed", "valid", true, object(managed)) diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index 1a6feee22..0fd31f1df 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -14,48 +14,22 @@ func Operations() providercontract.Operations { "Renew": {State: providercontract.Supported}, "Kill": {State: providercontract.Supported}, "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteSnapshot": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_preparation_cache"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_preparation_cache"}, "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } -func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} -} -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} -} -func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "GetCompute", Reason: Operations()["GetCompute"].Reason} -} -func (p *Provider) Suspend(context.Context, sandbox.SuspendRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Suspend", Reason: Operations()["Suspend"].Reason} -} -func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: Operations()["Resume"].Reason} -} -func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { - return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} -} -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} -} -func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { - return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} -} -func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} -} func (p *Provider) RestorePreparation(context.Context, sandbox.Reference, string) (bool, error) { return false, &providercontract.UnsupportedError{Operation: "RestorePreparation", Reason: Operations()["RestorePreparation"].Reason} } diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index e420a406c..ef4e2dc50 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -22,6 +22,8 @@ type Config struct { Binary, StateDir, InstallationID, APIKey, Template, APIURL, Domain string TimeoutSeconds int Resources *sandbox.Resources + // Compatibility is derived from the other fields by NewWithCaller. + Compatibility sandbox.CheckpointCompatibility } // Discover uses the same pinned SDK helper without requiring a saved deployment. @@ -149,6 +151,7 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { resources := *c.Resources c.Resources = &resources } + c.Compatibility = c.compatibility() return &Provider{config: c, caller: caller, now: time.Now}, nil } @@ -194,12 +197,18 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen } connection = &value } - out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) + return p.exchange(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) +} + +// exchange sends one validated helper request and maps its sanitized outcome. +func (p *Provider) exchange(ctx context.Context, q Request) (Response, error) { + operation, r := q.Operation, q.Reference + out, err := p.caller.Call(ctx, q) if errors.Is(err, errHelperNotStarted) { return unstarted(operation, r), sandbox.ErrComputeUnconfirmed } if err != nil || out.Version != ProtocolVersion { - if operation == "command" { + if operation == "command" || operation == "command_compute" { return Response{}, sandbox.ErrCommandUnconfirmed } return Response{}, sandbox.ErrComputeUnconfirmed @@ -211,7 +220,7 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen case "": return out, nil case "template_invalid": - return out, fmt.Errorf("%w: This E2B template lacks the current Runtime startup entry point. Build a template with this release's build-template.py and select it in the sandbox deployment.", sandbox.ErrInvalid) + return out, fmt.Errorf("%w: This E2B template lacks the current Runtime startup entry point or was built for another Runtime bootstrap version. Build a template with this release's build-template.py and select it in the sandbox deployment.", sandbox.ErrInvalid) case "team_mismatch": return out, sandbox.ErrCredentialOwnership case "unauthorized": @@ -224,6 +233,8 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen return out, sandbox.ErrExists case "not_found": return out, sandbox.ErrNotFound + case "checkpoint_unusable": + return out, sandbox.ErrCheckpointUnusable case "command_unconfirmed": return out, sandbox.ErrCommandUnconfirmed default: @@ -286,16 +297,18 @@ func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { } return err } -func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { +func validCommand(c sandbox.Command) error { if len(c.Args) == 0 || len(c.Stdin) > sandbox.MaxCommandInputBytes || c.Directory != "" && !filepath.IsAbs(c.Directory) { - return sandbox.CommandResult{}, sandbox.ErrInvalid + return sandbox.ErrInvalid } for _, arg := range c.Args { if strings.ContainsRune(arg, 0) { - return sandbox.CommandResult{}, sandbox.ErrInvalid + return sandbox.ErrInvalid } } - out, err := p.call(ctx, "command", r, nil, &c) + return nil +} +func commandResult(out Response, err error) (sandbox.CommandResult, error) { if err != nil { return sandbox.CommandResult{}, err } @@ -304,6 +317,12 @@ func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbo } return *out.Command, nil } +func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { + if err := validCommand(c); err != nil { + return sandbox.CommandResult{}, err + } + return commandResult(p.call(ctx, "command", r, nil, &c)) +} // A fresh allocation Create rejected before process startup has no cloud effects. // The common Provider contract forbids replaying an earlier unknown Create. diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index 14f788411..acb175140 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -19,6 +19,7 @@ func Operations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "RestorePreparation": {State: providercontract.Supported}, "SavePreparation": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index 37637de66..0053b5295 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -261,6 +261,11 @@ func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Com return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) } +// RenewCompute observes: a microsandbox VM has no native lease. +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { + return p.GetCompute(ctx, r, c) +} + func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err diff --git a/services/core/internal/sandbox/node/agent.go b/services/core/internal/sandbox/node/agent.go index 6508929cd..4b3118e51 100644 --- a/services/core/internal/sandbox/node/agent.go +++ b/services/core/internal/sandbox/node/agent.go @@ -402,7 +402,7 @@ func (a *agent) executeWork(ctx context.Context, task work) { func logOperation(ctx context.Context, task work, started, finished time.Time, code string, err error) { level := slog.LevelInfo switch task.request.Operation { - case "info", "renew", "initial", "new_compute", "compute", "observe": + case "info", "renew", "initial", "new_compute", "compute", "renew_compute", "observe": if code == "" { level = slog.LevelDebug } diff --git a/services/core/internal/sandbox/node/generations.go b/services/core/internal/sandbox/node/generations.go index 89e3add8d..132720dad 100644 --- a/services/core/internal/sandbox/node/generations.go +++ b/services/core/internal/sandbox/node/generations.go @@ -354,7 +354,7 @@ func (m *GenerationManager) probeLoop() { } ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) checkpoint, err := g.value.Probe(ctx) - if err == nil && ((sandbox.SupportsCheckpoint(g.value.Provider) && (checkpoint == nil || checkpoint.Validate() != nil)) || (!sandbox.SupportsCheckpoint(g.value.Provider) && checkpoint != nil)) { + if err == nil && ((sandbox.SupportsCheckpoint(g.value.Provider.ProviderOperations()) && (checkpoint == nil || checkpoint.Validate() != nil)) || (!sandbox.SupportsCheckpoint(g.value.Provider.ProviderOperations()) && checkpoint != nil)) { err = sandbox.ErrInvalid } cancel() diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 113447028..5c360e01d 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -169,7 +169,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { h := newTestHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), docker.Operations(), 1) - if sandbox.SupportsCheckpoint(p) { + if sandbox.SupportsCheckpoint(p.ProviderOperations()) { t.Fatal("docker advertised checkpoint") } _, err := p.GetInfo(context.Background(), reference()) diff --git a/services/core/internal/sandbox/node/operations.go b/services/core/internal/sandbox/node/operations.go index 55e9ba504..e62b13f56 100644 --- a/services/core/internal/sandbox/node/operations.go +++ b/services/core/internal/sandbox/node/operations.go @@ -16,6 +16,7 @@ var operationMethods = map[string]string{ "delete_snapshot": "DeleteSnapshot", "command_compute": "RunCommandCompute", "resume_compute": "ResumeCompute", + "renew_compute": "RenewCompute", "restore_preparation": "RestorePreparation", "save_preparation": "SavePreparation", "observe": "Observe", diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 07bfec1e1..10fe1f6d0 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -24,6 +24,7 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -56,6 +57,9 @@ func (*fakeProvider) RunCommandCompute(context.Context, sandbox.Reference, sandb func (*fakeProvider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: "fixture_operation_not_supported"} } +func (*fakeProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} func (*fakeProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } @@ -75,6 +79,7 @@ func (*observationProvider) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index 4a8b5b34c..07d8b5618 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -166,6 +166,10 @@ func (p *provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c san return p.state(ctx, request{Operation: "resume_compute", Reference: r, Compute: &c}) } +func (p *provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, request{Operation: "renew_compute", Reference: r, Compute: &c}) +} + // GenerationProvider routes every operation with allocation-owned generation, // distinct from the request's compute generation. declared is the kind's // registered operations. diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index c12b3cda0..b024a229c 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -19,7 +19,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 7 +const ProtocolVersion = 8 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 @@ -250,7 +250,7 @@ func (q request) validate() error { if count == 0 || count == 1 && q.Snapshot != nil { return nil } - case "compute", "kill_compute", "resume_compute": + case "compute", "kill_compute", "resume_compute", "renew_compute": if count == 1 && q.Compute != nil { return nil } @@ -345,6 +345,9 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) (respons case "resume_compute": state, err = p.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state + case "renew_compute": + state, err = p.RenewCompute(ctx, q.Reference, *q.Compute) + out.State = &state case "command_compute": command, err = p.RunCommandCompute(ctx, q.Reference, *q.Compute, *q.Command) out.Command = &command diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 017965442..9b6363d81 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -50,11 +50,6 @@ func ValidateRegistration(a Adapter) error { if err := sandbox.ValidateOperations(operations); err != nil { return err } - // The common lifecycle suspends only node allocations, so only a nodes - // registration may declare checkpoint support. - if operations["Initial"].State == providercontract.Supported && a.Mode != sandbox.DeploymentNodes { - return invalid("checkpoint support outside nodes mode") - } if a.Policy.DefaultResources != nil && a.Policy.DefaultResources.ValidatePolicy("", a.Policy) != nil { return invalid("default resources") } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 363720fcd..4d7c10cf2 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -162,13 +162,13 @@ func TestRegistrationRejectsInvalidDefaultResources(t *testing.T) { } } -// Only a nodes registration may declare checkpoint support. -func TestRegistrationCheckpointRequiresNodes(t *testing.T) { +// Checkpoint support follows the declaration in both modes. +func TestRegistrationAdmitsDirectCheckpoint(t *testing.T) { registry := Builtin() a := registry.adapters["microsandbox"] a.Mode, a.BuildLocal, a.NodeArtifacts, a.BuildDirect = "direct", nil, nil, registry.adapters["e2b"].BuildDirect - if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) || !strings.Contains(err.Error(), "checkpoint") { - t.Fatal("direct checkpoint registration accepted", err) + if err := ValidateRegistration(a); err != nil { + t.Fatal("direct checkpoint registration rejected", err) } } diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index f90ce974c..d797896ed 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -90,7 +90,7 @@ func (r *Registry) SupportsCheckpoint(kind string) (bool, error) { if err != nil { return false, err } - return a.Operations()["Initial"].State == providercontract.Supported, nil + return sandbox.SupportsCheckpoint(a.Operations()), nil } // RetainedLimit keeps nodes without checkpoint support within their active capacity. @@ -99,7 +99,7 @@ func (r *Registry) RetainedLimit(kind string, active, retained int) (int, error) if err != nil { return 0, err } - if a.Mode == sandbox.DeploymentNodes && a.Operations()["Initial"].State != providercontract.Supported { + if a.Mode == sandbox.DeploymentNodes && !sandbox.SupportsCheckpoint(a.Operations()) { return active, nil } return retained, nil diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 47a0629bb..10f5be9ba 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -19,7 +19,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { }{ {"docker", "nodes", "nodes", false}, {"microsandbox", "nodes", "nodes", true}, - {"e2b", "direct", "e2b", false}, + {"e2b", "direct", "e2b", true}, } { t.Run(tc.kind, func(t *testing.T) { d, err := registry.Describe(tc.kind, installation) diff --git a/services/core/internal/sandbox/runtime_bootstrap.go b/services/core/internal/sandbox/runtime_bootstrap.go index a47981658..ca8e3e34a 100644 --- a/services/core/internal/sandbox/runtime_bootstrap.go +++ b/services/core/internal/sandbox/runtime_bootstrap.go @@ -9,5 +9,6 @@ func (b Bootstrap) RuntimeConnection() runtimebootstrap.Connection { Version: runtimebootstrap.Version, CoreURL: b.CoreURL, DeviceID: b.DeviceID, Credential: b.Credential, EnvironmentID: b.EnvironmentID, SessionID: b.SessionID, Workspace: runtimebootstrap.RuntimeWorkspace, + SuspendControl: b.SuspendControl, } } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 2e0a563a4..622b8bd69 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -62,6 +62,8 @@ type Bootstrap struct { Workspace *workspacefs.Binding `json:",omitempty"` Reference SessionID, DeviceID, CoreURL, Credential string + // SuspendControl is true exactly when the Provider declares checkpoint support. + SuspendControl bool } // Info describes compute only. Running does not establish daemon authentication, @@ -125,6 +127,10 @@ type SandboxProvider interface { RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) + // RenewCompute extends the native lease of the exact running incarnation + // where the Provider has one, and otherwise observes exactly like + // GetCompute. It never revives stopped compute. + RenewCompute(context.Context, Reference, Compute) (ComputeState, error) // The preparation cache keeps the Environment filesystem left by one finished // preparation. Key is opaque and the adapter owns storage and lifetime. Both @@ -143,7 +149,7 @@ var requiredOperations = []string{"Create", "GetInfo", "Renew", "Kill", "RunComm // checkpointOperations are all supported or all unsupported: partial cleanup or // restore support cannot safely own a compute incarnation. -var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute"} +var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "RenewCompute"} // preparationOperations are both supported or both unsupported. var preparationOperations = []string{"RestorePreparation", "SavePreparation"} @@ -268,7 +274,7 @@ func ValidateOperations(operations providercontract.Operations) error { } } for _, name := range checkpointOperations { - if operations[name].State != operations["Initial"].State { + if (operations[name].State == providercontract.Supported) != SupportsCheckpoint(operations) { return fmt.Errorf("%w: incomplete checkpoint lifecycle", providercontract.ErrContract) } } @@ -278,8 +284,10 @@ func ValidateOperations(operations providercontract.Operations) error { return nil } -func SupportsCheckpoint(p SandboxProvider) bool { - return providercontract.Require(p, "Initial") == nil +// SupportsCheckpoint is the one predicate over a declaration: the checkpoint +// operations are all supported or all unsupported, so Initial decides. +func SupportsCheckpoint(operations providercontract.Operations) bool { + return operations["Initial"].State == providercontract.Supported } // ProcessPaths locates installed adapter helpers and their private state. The diff --git a/services/core/tests/integration/provider_operations_fixture_test.go b/services/core/tests/integration/provider_operations_fixture_test.go index c7d052b56..c68ca0dc0 100644 --- a/services/core/tests/integration/provider_operations_fixture_test.go +++ b/services/core/tests/integration/provider_operations_fixture_test.go @@ -24,6 +24,7 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -56,6 +57,9 @@ func (*lifecycleProvider) RunCommandCompute(context.Context, sandbox.Reference, func (*lifecycleProvider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: "fixture_operation_not_supported"} } +func (*lifecycleProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} func (*lifecycleProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } @@ -75,6 +79,7 @@ func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations "DeleteSnapshot": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 9b164bd5b..2c93db85e 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -38,7 +38,7 @@ type fakeCheckpointProvider struct { captures, restores, captureObservations, restoreObservations int computeKills, snapshotDeletes, wakeCommands int promptFrames atomic.Int32 - quiesces, resumes atomic.Int32 + quiesces, resumes, renews atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() } @@ -162,6 +162,10 @@ func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Ref delete(p.snapshots, s.OperationID) return nil } +func (p *fakeCheckpointProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renews.Add(1) + return p.GetCompute(ctx, r, c) +} func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { state, err := p.GetCompute(ctx, r, c) if err != nil { @@ -273,6 +277,12 @@ type computeLifecycleFixture struct { } func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *computeLifecycleFixture { + return newComputeLifecycleFixtureFor(t, "microsandbox", maxActive, maxRetained) +} + +// newComputeLifecycleFixtureFor runs the fixture on a nodes deployment, or on a +// direct one (e2b) where allocations have no node. +func newComputeLifecycleFixtureFor(t *testing.T, provider string, maxActive, maxRetained int) *computeLifecycleFixture { t.Helper() s, _ := newManagedTestStore(t) registry := runtimegateway.NewRegistry() @@ -288,13 +298,15 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu } server.Close() }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{Retention: time.Hour}} + f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, provider), policy: execution.RuntimeSuspensionPolicy{Retention: time.Hour}} view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } f.sql(`UPDATE runtime_deployment SET idle_seconds=30`) - f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + if provider == "microsandbox" { + f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + } f.start() return f } @@ -303,7 +315,9 @@ func (f *computeLifecycleFixture) start() { t.Helper() w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) // The Worker's claim starts a new owner epoch, in which the node reconnects. - onlineManagerNode(t, f.store, f.node) + if f.node != "" { + onlineManagerNode(t, f.store, f.node) + } var once sync.Once stop := func() { once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) @@ -392,6 +406,28 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T } } +// A direct allocation has no node and follows the same suspend and wake flow. +func TestRuntimeComputeLifecycleDirectIdleSuspendAndWake(t *testing.T) { + f := newComputeLifecycleFixtureFor(t, "e2b", 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + suspended := f.phase(tenant, env.ID, "suspended") + if f.provider.renews.Load() == 0 { + t.Fatal("running compute was observed without renewing its native lease") + } + if f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.snapshots) != 1 || suspended.NodeID != "" { + t.Fatal("direct capture did not release source compute") + } + f.queued(suspended) + awake := f.phase(tenant, env.ID, "running") + if awake.ID != owner.ID || f.provider.creates != 1 || f.provider.restores != 1 { + t.Fatal("direct wake replaced the allocation") + } + if awake.NodeID != "" { + t.Fatal("direct allocation gained a node") + } +} + func TestRuntimeComputeLifecycleLostCaptureAndRestoreObserveWithoutReplay(t *testing.T) { f := newComputeLifecycleFixture(t, 1, 2) tenant, _, env, owner := f.create() diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index aedeca68c..c468b4c11 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -38,6 +38,11 @@ func (p *wakeHintScanProvider) GetCompute(ctx context.Context, reference sandbox return p.fakeCheckpointProvider.GetCompute(ctx, reference, compute) } +// A running allocation with a connected Runtime is observed by RenewCompute. +func (p *wakeHintScanProvider) RenewCompute(ctx context.Context, reference sandbox.Reference, compute sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, reference, compute) +} + type wakeHintIntegrationTarget struct { tenant string session sessions.Session diff --git a/services/core/tests/integration/sandbox_deployment_view_test.go b/services/core/tests/integration/sandbox_deployment_view_test.go index ca1ed70b7..9f8051f31 100644 --- a/services/core/tests/integration/sandbox_deployment_view_test.go +++ b/services/core/tests/integration/sandbox_deployment_view_test.go @@ -38,7 +38,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { for _, want := range []string{ `"specification":{"resources":{"cpus":2,"memory_mib":2048}}`, `"template_build":{"status":"ready","resources":{"cpus":2,"memory_mib":2048,"root_disk_mib":24063}}`, - `"suspension":null`, + `"suspension":{"idle_seconds":300,"retention_seconds":86400}`, } { if !bytes.Contains(raw, []byte(want)) { t.Fatalf("E2B view lacks %s: %s", want, raw) diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 07243a1cb..6672d41e0 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -17,6 +17,12 @@ The account key is stored encrypted in Core's database and is write-only. It rea | `renew` | `Renew` | Extends the lease of the running sandbox to the configured timeout, then rereads it | | `kill` | `Kill` | Destroys every matching sandbox and confirms that none remains | | `command` | `RunCommand` | Runs one bounded command as the Runtime user on a running sandbox whose bootstrap completed; output is limited to 1 MiB per stream | +| `compute`, `renew_compute` | `GetCompute`, `RenewCompute` | One sandbox spans every generation of an allocation, and the receipt records the current generation, the pause and the resume. Any other generation is `not_found`. `renew_compute` extends a running sandbox's lease and otherwise observes like `compute` | +| `suspend` | `Suspend` | Pauses the sandbox with the SDK's native memory snapshot, so the daemon survives and the paused sandbox is the snapshot. A pause that is dispatched but not seen complete is `unconfirmed`, never running | +| `resume` | `Resume` | Records the attempt as dispatched, then reconnects with `connect`; the same operation is never dispatched twice and a failure after dispatch is `unconfirmed`. A snapshot that does not match the paused sandbox, its endpoint or its template build is `checkpoint_unusable` before any native call, durably for the operation | +| `resume_compute` | `ResumeCompute` | Thaws the source only while it is running | +| `kill_compute`, `delete_snapshot` | `KillCompute`, `DeleteSnapshot` | Destroy the sandbox only for the generation that owns it; a superseded generation owns nothing | +| `command_compute` | `RunCommandCompute` | Runs one bounded command, such as the wake command, on the running current sandbox | | `validate_deployment` | Deployment setup | Reads the template's builds and requires the exact build to be ready with the configured CPU and memory. Without configured resources the selection adopts the build's CPU and memory. Returns the build's status, CPU, memory and reported disk for Core to record; bounded to 30 seconds | | `list_templates`, `list_builds` | [Configuration discovery](../../../../contracts/agents-api/sandbox-deployment.md#configuration-discovery) | Pages the key's visible templates (`GET /v2/templates`) or one template's ready builds, with a transient key. Results are capped at 200 and write no receipt | | `observe` | Runtime observations | One allocation; see [Observations](#observations) | @@ -50,7 +56,7 @@ An unknown Create is never repeated. A Create whose connection material was lost ## Inspection and cleanup -Inspection uses SDK metadata and ID reads only. SDK `connect` is never used because it can resume paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. +Inspection uses SDK metadata and ID reads only. SDK `connect` is used only in Resume, because it resumes paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. `CreateSettled` proves that the original Create and bootstrap can no longer mutate; it is independent of `BootstrapComplete`. An empty lookup never settles an unknown Create. Kill destroys every matching sandbox, confirms that none remains and only then records a settled tombstone; it returns `State=absent` with `CreateSettled`. A settled rejected Create with no sandbox IDs proves absence without a cloud request, so GetInfo and Kill still succeed when the key is invalid. Ordinary missing compute has no such proof. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index ee81a1327..bee1ee648 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,6 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" -ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] +ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] @@ -9,9 +9,11 @@ MAX_OUTPUT = 1048576 MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute","renew_compute","suspend","resume","kill_compute","delete_snapshot","command_compute","resume_compute"] PROTOCOL_VERSION = 1 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Snapshot","Deadline"] +RESPONSE_FIELDS = ["Version","Info","Command","State","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RUNTIME_BOOTSTRAP_VERSION = 3 SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 76557b500..bbf5091c2 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -1,4 +1,4 @@ -"""Five bounded SDK operations for an already authorized Core allocation, plus +"""Bounded SDK operations for an already authorized Core allocation, plus read-only deployment validation and observation.""" from concurrent.futures import ThreadPoolExecutor import json @@ -14,10 +14,26 @@ from sdk import connection_material, definitely_rejected, list_builds, list_templates, read_metrics, restore, run, sdk_options, validate_deployment, verify_team_template from state import Failure, Receipt, private_root, read_receipt from helper_contract_generated import (PROTOCOL_VERSION, OPERATIONS, REQUEST_FIELDS, REFERENCE_FIELDS, - MAX_CREDENTIAL_REFERENCES, MANAGED_BOOTSTRAP_FIELDS, MANAGED_BOOTSTRAP_REQUIRED_FIELDS) + MAX_CREDENTIAL_REFERENCES, MANAGED_BOOTSTRAP_FIELDS, MANAGED_BOOTSTRAP_REQUIRED_FIELDS, RUNTIME_BOOTSTRAP_VERSION) PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +PENDING_PAUSE_SETTLE_SECONDS = 600 # a pause request settles in seconds; one still unpaused after ten minutes is taken as not applied + +# Exits 78 unless the template carries the startup script and a contract for this Runtime bootstrap version. +TEMPLATE_CHECK = """import importlib.util, os, sys +root, version = sys.argv[1], int(sys.argv[2]) +script = os.path.join(root, 'managed_init.py') +try: + spec = importlib.util.spec_from_file_location('template_contract', os.path.join(root, 'helper_contract_generated.py')) + contract = importlib.util.module_from_spec(spec) + spec.loader.exec_module(contract) + current = getattr(contract, 'RUNTIME_BOOTSTRAP_VERSION', None) +except Exception: + current = None +sys.exit(78 if not os.path.isfile(script) or not os.access(script, os.R_OK) or current != version else 0) +""" +COMPUTE_OPERATIONS = ('compute', 'renew_compute', 'resume_compute', 'suspend', 'resume', 'kill_compute', 'delete_snapshot', 'command_compute') def valid_id(value): @@ -175,7 +191,7 @@ def client(self, cloud): raise Failure('unconfirmed') return restore(material, self.options()) - def inspect(self): + def inspect(self, qualify=True): if self.rejected_absence(): return None found = self.discover() @@ -186,7 +202,8 @@ def inspect(self): if len(found) != 1: raise Failure('unconfirmed') cloud = found[0] - self.qualified(cloud) + if qualify: + self.qualified(cloud) record = self.receipt.data or {} if (cloud.state == 'running' and not record.get('bootstrap_complete') and record.get('connection') and record.get('status') not in ('bootstrap_failed', 'killed')): @@ -233,15 +250,14 @@ def create(self): self.receipt.save(status='configuration_rejected', settled=True) raise # Validate the current template entry point before writing any credential. - check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', - "import os,sys; sys.exit(78 if not os.path.isfile('/opt/oac-e2b/managed_init.py') or not os.access('/opt/oac-e2b/managed_init.py', os.R_OK) else 0)"]}, + check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', TEMPLATE_CHECK, '/opt/oac-e2b', str(RUNTIME_BOOTSTRAP_VERSION)]}, self.remaining, user='root') if check['ExitCode'] != 0: self.receipt.save(status='bootstrap_failed', settled=True) raise Failure('template_invalid' if check['ExitCode'] == 78 else 'unconfirmed') payload = dict(bootstrap, InstallationID=self.config['InstallationID'], RuntimeBootstrap=self.q['RuntimeBootstrap']) - del payload['CoreURL'], payload['Credential'] + del payload['CoreURL'], payload['Credential'], payload['SuspendControl'] if (set(payload) - set(MANAGED_BOOTSTRAP_FIELDS) or not set(MANAGED_BOOTSTRAP_REQUIRED_FIELDS) <= set(payload)): raise Failure('invalid') @@ -312,6 +328,168 @@ def observe(self): return {'Status': 'unavailable'} return observed(found[0], point) + # One sandbox spans every generation. The receipt's generation says which + # incarnation owns it: pause leaves it as the snapshot and resume advances it. + def generation(self): + return (self.receipt.data or {}).get('generation', 0) + + def current(self, compute): + """The sandbox of the exact current incarnation; any other generation is absent.""" + if compute['Generation'] != self.generation(): + raise Failure('not_found') + cloud = self.inspect() + if cloud is None: + raise Failure('not_found') + if compute.get('ID') and compute['ID'] != cloud.sandbox_id: + raise Failure('ownership') + return cloud + + def state(self, compute, cloud, **extra): + status = getattr(cloud.state, 'value', cloud.state) + return dict(Compute=dict(compute, ID=cloud.sandbox_id), Status=status, + BootstrapComplete=bool((self.receipt.data or {}).get('bootstrap_complete')), **extra) + + def lease(self, cloud): + Sandbox.set_timeout(cloud.sandbox_id, self.config['TimeoutSeconds'], **self.options()) + return self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) + + def get_compute(self): + compute = self.q['Compute'] + return self.state(compute, self.current(compute)) + + def renew_compute(self): + compute = self.q['Compute'] + cloud = self.current(compute) + if cloud.state == 'running': + cloud = self.lease(cloud) + return self.state(compute, cloud) + + def resume_compute(self): + """Thaw only a running source; a paused sandbox is a snapshot, not a source.""" + compute = self.q['Compute'] + cloud = self.current(compute) + if cloud.state != 'running': + raise Failure('unconfirmed') + return self.state(compute, self.lease(cloud)) + + def command_compute(self): + cloud = self.current(self.q['Compute']) + if cloud.state != 'running' or not self.receipt.data.get('bootstrap_complete'): + raise Failure('unconfirmed') + return run(self.client(cloud), self.q['Command'], self.remaining) + + def snapshot(self, operation, source, cloud): + return {'Compatibility': self.config['Compatibility'], 'Reference': source['Name'] + ':s-' + operation, + 'ID': cloud.sandbox_id, 'Digest': '', 'CheckpointID': '', 'CheckpointRoot': '', + 'OperationID': operation, 'SourceGeneration': source['Generation'], 'SourceName': source['Name'], + 'SourceID': cloud.sandbox_id} + + def suspend(self): + request = self.q['Suspend'] + source, operation = request['Source'], request['OperationID'] + cloud = self.current(source) + if cloud.state == 'running' and not request['ObserveOnly']: + # Record the attempt before the native call; pause of a paused sandbox is a no-op. + self.receipt.save(pause={'operation_id': operation, 'status': 'pending', 'at': time.time()}) + Sandbox.pause(cloud.sandbox_id, **self.options()) + cloud = self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) + pause = (self.receipt.data or {}).get('pause') or {} + if cloud.state != 'paused' or pause.get('operation_id') != operation: + # A dispatched pause may still take effect, so running is never reported for it. + pending = pause.get('operation_id') == operation and pause.get('status') == 'pending' + if pending and request['ObserveOnly'] and cloud.state == 'running' and time.time() - pause.get('at', time.time()) > PENDING_PAUSE_SETTLE_SECONDS: + # Observation proves the long-dispatched pause did not take effect; close it as not taken. + self.receipt.save(pause={'operation_id': operation, 'status': 'not_taken'}) + return self.state(source, cloud) + if request['ObserveOnly'] and not pending: + return self.state(source, cloud) + raise Failure('unconfirmed') + self.receipt.save(pause={'operation_id': operation, 'status': 'paused'}) + return self.state(source, cloud, Snapshot=self.snapshot(operation, source, cloud), SourceStopped=True) + + def close_resume(self, operation, kind): + data = self.receipt.data or {} + closed = dict(data.get('closed', {}), **{operation: kind}) + self.receipt.save(closed=dict(list(closed.items())[-32:]), resume=None) + + def closed_state(self, target, operation): + return {'Compute': target, 'Status': 'absent', 'BootstrapComplete': False, 'RestoreAttemptClosed': operation} + + def finish_resume(self, target, operation, cloud, material=None): + values = {'generation': target['Generation'], 'resume': {'operation_id': operation, 'status': 'resumed'}} + if material is not None: + values['connection'] = material + self.receipt.save(**values) + return self.state(target, cloud) + + def resume(self): + request = self.q['Resume'] + operation, snapshot, target = request['OperationID'], request['Snapshot'], request['Target'] + data = self.receipt.data or {} + kind = data.get('closed', {}).get(operation) + if kind == 'unusable': + raise Failure('checkpoint_unusable') + if kind: + return self.closed_state(target, operation) + record = data.get('resume') or {} + if record.get('operation_id') == operation: + # Native restoration was dispatched: observe it, never dispatch again. + cloud = self.inspect() + if cloud is None: + raise Failure('not_found') + if cloud.state != 'running': + raise Failure('unconfirmed') + return self.finish_resume(target, operation, cloud) + if request['ObserveOnly']: + # No dispatch is recorded, so this attempt can still be closed durably. + self.close_resume(operation, 'attempt') + return self.closed_state(target, operation) + pause = data.get('pause') or {} + # Drift is judged below, before dispatch, as an unusable checkpoint. + cloud = self.inspect(qualify=False) + usable = (cloud is not None and cloud.state == 'paused' and pause.get('status') == 'paused' and + pause.get('operation_id') == snapshot['OperationID'] and + snapshot['SourceGeneration'] == self.generation() < target['Generation'] and + snapshot['SourceID'] == cloud.sandbox_id) + if usable: + try: + self.qualified(cloud) + except Failure: + usable = False + if not usable: + self.close_resume(operation, 'unusable') + raise Failure('checkpoint_unusable') + self.receipt.save(resume={'operation_id': operation, 'target': target['Generation'], 'status': 'dispatched'}) + try: + connected = Sandbox.connect(cloud.sandbox_id, timeout=self.config['TimeoutSeconds'], **self.options()) + except SandboxNotFoundException: + # Dispatched: the outcome is unknown and the attempt stays open. + raise Failure('unconfirmed') from None + cloud = self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) + if cloud.state != 'running': + raise Failure('unconfirmed') + return self.finish_resume(target, operation, cloud, connection_material(connected)) + + def kill_compute(self): + """Kill only the sandbox's owning incarnation; a superseded one owns nothing.""" + compute = self.q['Compute'] + data = self.receipt.data or {} + record = data.get('resume') or {} + if compute['Generation'] == self.generation() or (record.get('status') == 'dispatched' and record.get('target') == compute['Generation']): + self.kill_owned(compute.get('ID')) + + def delete_snapshot(self): + """A snapshot is the paused sandbox itself; one consumed by resume owns nothing.""" + snapshot = self.q['Snapshot'] + if snapshot['SourceGeneration'] == self.generation(): + self.kill_owned(snapshot['SourceID']) + + def kill_owned(self, sandbox_id): + ids = (self.receipt.data or {}).get('ids', []) + if sandbox_id and ids and sandbox_id not in ids: + raise Failure('ownership') + self.kill() + def metadata_for(self, reference): return {PREFIX + field.lower(): value for field, value in dict(reference, InstallationID=self.config['InstallationID']).items()} @@ -391,12 +569,25 @@ def execute(self): raise Failure('unconfirmed') result = run(self.client(cloud), self.q['Command'], self.remaining) return {'Version': PROTOCOL_VERSION, 'Command': result, 'ErrorCode': ''} + if operation in ('compute', 'renew_compute', 'resume_compute', 'suspend', 'resume'): + handler = {'compute': self.get_compute, 'renew_compute': self.renew_compute, 'resume_compute': self.resume_compute, + 'suspend': self.suspend, 'resume': self.resume}[operation] + return {'Version': PROTOCOL_VERSION, 'State': handler(), 'ErrorCode': ''} + if operation == 'command_compute': + return {'Version': PROTOCOL_VERSION, 'Command': self.command_compute(), 'ErrorCode': ''} + if operation in ('kill_compute', 'delete_snapshot'): + {'kill_compute': self.kill_compute, 'delete_snapshot': self.delete_snapshot}[operation]() + return {'Version': PROTOCOL_VERSION, 'ErrorCode': ''} cloud = {'create': self.create, 'inspect': self.inspect, 'renew': self.renew}[operation]() return {'Version': PROTOCOL_VERSION, 'Info': self.info(cloud, absent=cloud is None), 'ErrorCode': ''} except Failure as error: + if self.q['Operation'] in COMPUTE_OPERATIONS: + return {'Version': PROTOCOL_VERSION, 'ErrorCode': error.code} info = self.info() if error.code == 'not_found': info = dict(self.reference, ProviderID='', State='absent', BootstrapComplete=False, CreateSettled=False) return {'Version': PROTOCOL_VERSION, 'Info': info, 'ErrorCode': error.code} except Exception: + if self.q['Operation'] in COMPUTE_OPERATIONS: + return {'Version': PROTOCOL_VERSION, 'ErrorCode': 'unconfirmed'} return {'Version': PROTOCOL_VERSION, 'Info': self.info(), 'ErrorCode': 'unconfirmed'} diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 3d22f01e1..b2a3e7877 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -3,7 +3,10 @@ from datetime import datetime, timedelta, timezone import json from pathlib import Path +import subprocess +import sys import tempfile +import time from types import SimpleNamespace import unittest from unittest.mock import Mock, patch @@ -12,12 +15,13 @@ from e2b import SandboxState from e2b.exceptions import AuthenticationException, SandboxNotFoundException +import provider from provider import Provider from sdk import restore, run from state import Failure, Receipt -class ProviderTest(unittest.TestCase): +class Fixture(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) @@ -27,9 +31,9 @@ def setUp(self): self.request = {'Version': 1, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), - CoreURL='https://core.example/api/v1', Credential='private-runtime-secret')} + CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', SuspendControl=True)} self.request['RuntimeBootstrap'] = { - 'version': 2, 'core_url': self.request['Bootstrap']['CoreURL'], + 'version': 3, 'suspend_control': True, 'core_url': self.request['Bootstrap']['CoreURL'], 'device_id': self.request['Bootstrap']['DeviceID'], 'credential': self.request['Bootstrap']['Credential']} self.cloud = Mock(sandbox_id='owned-id', sandbox_domain='e2b.app', _envd_version='0.5.0', @@ -60,6 +64,8 @@ def call(self, operation): def record(self): return json.loads(next(Path(self.temporary.name).glob('*.json')).read_text()) + +class ProviderTest(Fixture): def test_create_recover_and_never_replay(self): result = self.call('create') self.assertEqual(result['ErrorCode'], '') @@ -322,5 +328,120 @@ def test_unknown_command_does_not_replay(self): client.commands.run.assert_called_once() +class TemplateCheckTest(unittest.TestCase): + def exit_code(self, files): + with tempfile.TemporaryDirectory() as root: + for name, text in files.items(): + Path(root, name).write_text(text) + return subprocess.run([sys.executable, '-I', '-c', provider.TEMPLATE_CHECK, root, '3']).returncode + + def test_template_needs_startup_script_and_current_bootstrap_version(self): + self.assertEqual(self.exit_code({'managed_init.py': '', 'helper_contract_generated.py': 'RUNTIME_BOOTSTRAP_VERSION = 3'}), 0) + self.assertEqual(self.exit_code({'managed_init.py': '', 'helper_contract_generated.py': 'RUNTIME_BOOTSTRAP_VERSION = 2'}), 78) + self.assertEqual(self.exit_code({'managed_init.py': '', 'helper_contract_generated.py': ''}), 78) + self.assertEqual(self.exit_code({'helper_contract_generated.py': 'RUNTIME_BOOTSTRAP_VERSION = 3'}), 78) + + +class CheckpointTest(Fixture): + """One sandbox spans generations: pause keeps it and connect resumes it.""" + + def setUp(self): + super().setUp() + self.assertEqual(self.call('create')['ErrorCode'], '') + self.config['Compatibility'] = {'artifact_domain': 'a', 'execution_class': 'e'} + self.op = str(uuid4()) + + def compute(self, generation=0, restored=None): + return {'Generation': generation, 'Name': 'n%d' % generation, 'ID': 'owned-id', 'RestoredFrom': restored} + + def suspend(self, observe=False): + self.request['Suspend'] = {'Reference': self.reference, 'OperationID': self.op, 'Source': self.compute(), + 'Snapshot': None, 'ObserveOnly': observe} + return self.call('suspend') + + def resume(self, snapshot, observe=False, op=None): + target = self.compute(1, snapshot) + self.request['Resume'] = {'Reference': self.reference, 'OperationID': op or self.op, 'Snapshot': snapshot, + 'Target': target, 'ObserveOnly': observe} + return self.call('resume') + + def paused(self): + self.cloud.state = SandboxState.PAUSED + + def test_suspend_pauses_once_and_resume_advances_the_generation(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + result = self.suspend() + self.assertEqual(result['ErrorCode'], '') + snapshot = result['State']['Snapshot'] + self.assertTrue(result['State']['SourceStopped']) + self.assertEqual(result['State']['Status'], 'paused') + self.assertEqual(self.suspend()['State']['Snapshot'], snapshot) + self.api.pause.assert_called_once() + self.api.connect.side_effect = lambda *a, **k: (setattr(self.cloud, 'state', 'running'), self.cloud)[1] + resumed = self.resume(snapshot) + self.assertEqual(resumed['ErrorCode'], '') + self.assertEqual(resumed['State']['Status'], 'running') + self.assertEqual(self.record()['generation'], 1) + self.api.connect.assert_called_once() + self.resume(snapshot) + self.api.connect.assert_called_once() + + def test_superseded_generation_owns_nothing(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + self.api.connect.side_effect = lambda *a, **k: (setattr(self.cloud, 'state', 'running'), self.cloud)[1] + self.resume(snapshot) + self.request['Snapshot'] = snapshot + self.assertEqual(self.call('delete_snapshot')['ErrorCode'], '') + self.request['Compute'] = self.compute() + self.assertEqual(self.call('compute')['ErrorCode'], 'not_found') + self.assertEqual(self.call('kill_compute')['ErrorCode'], '') + self.api.kill.assert_not_called() + + def test_unusable_snapshot_is_durable_and_never_connects(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + other = str(uuid4()) + self.assertEqual(self.resume(dict(snapshot, OperationID=other))['ErrorCode'], 'checkpoint_unusable') + self.assertEqual(self.resume(snapshot)['ErrorCode'], 'checkpoint_unusable') + self.api.connect.assert_not_called() + + def test_unseen_pause_never_reports_running(self): + self.assertEqual(self.suspend()['ErrorCode'], 'unconfirmed') + self.assertEqual(self.suspend(observe=True)['ErrorCode'], 'unconfirmed') + + def test_drift_before_dispatch_is_unusable(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + self.cloud.template_id = 'drifted' + self.assertEqual(self.resume(snapshot)['ErrorCode'], 'checkpoint_unusable') + self.api.connect.assert_not_called() + + def test_failure_after_dispatch_stays_open(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + self.api.connect.side_effect = SandboxNotFoundException() + self.assertEqual(self.resume(snapshot)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.record()['resume']['status'], 'dispatched') + self.assertNotIn(self.op, self.record().get('closed', {})) + + def test_old_pending_pause_is_settled_by_observation(self): + self.assertEqual(self.suspend()['ErrorCode'], 'unconfirmed') + self.assertEqual(self.suspend(observe=True)['ErrorCode'], 'unconfirmed') + with patch('provider.time.time', return_value=time.time() + provider.PENDING_PAUSE_SETTLE_SECONDS + 1): + result = self.suspend(observe=True) + self.assertEqual((result['ErrorCode'], result['State']['Status']), ('', 'running')) + self.assertNotIn('Snapshot', result['State']) + self.assertEqual(self.record()['pause']['status'], 'not_taken') + + def test_renew_compute_extends_only_a_running_lease(self): + self.request['Compute'] = self.compute() + self.assertEqual(self.call('renew_compute')['ErrorCode'], '') + self.api.set_timeout.assert_called_once() + self.paused() + self.assertEqual(self.call('renew_compute')['State']['Status'], 'paused') + self.api.set_timeout.assert_called_once() + + if __name__ == '__main__': unittest.main() diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 204cb3e29..f04757e41 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -1,46 +1,55 @@ [ -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllocationID","payload":{"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllocationID","payload":{"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, {"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, -{"kind":"managed","name":"valid","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, -{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":true},"valid":false}, -{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":null},"valid":false}, -{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":2},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, +{"kind":"managed","name":"valid","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, +{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"command_compute","payload":{"Version":1,"Operation":"command_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute","payload":{"Version":1,"Operation":"compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"delete_snapshot","payload":{"Version":1,"Operation":"delete_snapshot","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill_compute","payload":{"Version":1,"Operation":"kill_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"renew_compute","payload":{"Version":1,"Operation":"renew_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume","payload":{"Version":1,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Target":{"Generation":1,"Name":"fixture-g1","ID":"","RestoredFrom":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"}},"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume_compute","payload":{"Version":1,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"suspend","payload":{"Version":1,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Source":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Snapshot":null,"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":true},"valid":false}, +{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":null},"valid":false}, +{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":"1"},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":2},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, {"kind":"response","name":"error-","payload":{"Version":1,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-checkpoint_unusable","payload":{"Version":1,"ErrorCode":"checkpoint_unusable"},"valid":true}, {"kind":"response","name":"error-command_unconfirmed","payload":{"Version":1,"ErrorCode":"command_unconfirmed"},"valid":true}, {"kind":"response","name":"error-exists","payload":{"Version":1,"ErrorCode":"exists"},"valid":true}, {"kind":"response","name":"error-invalid","payload":{"Version":1,"ErrorCode":"invalid"},"valid":true}, diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index e8dec91df..ac6bac559 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -22,7 +22,7 @@ Create names the VM from a hash of the installation and allocation reference plu Workspace, staging and outputs share the `/environment` filesystem, which keeps the Runtime's cross-device and link checks intact; the layered root filesystem can report different device IDs for a directory and its upper-layer files, so it holds no workspace data. The filesystem provider owns the directory; sandbox removal never deletes it. The binding and resource rules belong to [Workspace filesystem protocol](../../../../docs/workspace-provider.md#attachment). The [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) owns the Runtime directory layout. -VM creation does not run the image's entry point. The bootstrap runs as root through confidential standard input, with a two-minute limit. It creates the Runtime directories and the private control directory `/run/oac` (mode 0700, owned by UID 1000), writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json`, bind-mounts `/environment/workspace` at `/workspace` and starts `oac-daemon connect --bootstrap-file` in the background as UID/GID 1000. `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE=/run/oac/daemon-suspend.json` enables the daemon's park and wake control. The `io.oac.bootstrap` label then changes from `pending` to `complete` through the SDK's next-start modification policy, because v0.7.8 cannot update the labels of a running VM. That label confirms only these writes and the launch, not authentication or native readiness. +VM creation does not run the image's entry point. The bootstrap runs as root through confidential standard input, with a two-minute limit. It creates the Runtime directories and the private suspend control directory, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json`, bind-mounts `/environment/workspace` at `/workspace` and starts `oac-daemon connect --bootstrap-file` in the background as UID/GID 1000. The `io.oac.bootstrap` label then changes from `pending` to `complete` through the SDK's next-start modification policy, because v0.7.8 cannot update the labels of a running VM. That label confirms only these writes and the launch, not authentication or native readiness. A helper response carries `CreateSettled` with a configuration rejection only after native Create has completed, the first inspection has verified the exact compute ID and ownership, and the resource check has rejected the VM before bootstrap started. The adapter keeps the original error and validates the compute identity before passing the proof to Core. The private `initial_info` operation used by `GetInfo` and `Renew` can also return an exact initial `State` with `Status="absent"`, no native ID and `CreateSettled=true`, after typed native absence and durable closure of initial Create admission under the allocation lock. It accepts only generation zero without restored ancestry or a native ID. The adapter validates the complete receipt and returns absence without an error. Ordinary `inspect` used by `GetCompute`, uncertain Create outcomes, timeouts and ownership failures never produce this receipt; missing compute alone is not settlement. @@ -53,11 +53,11 @@ The same lock file fences a Create helper that has not yet acquired its lock, in `RunCommand` and `RunCommandCompute` pass standard input on an anonymous pipe, run as UID 1000 with a deadline and a 1 MiB limit per output stream, and return a result only after the input is fully written and the guest reports its exit. Timeouts, output overflow and missing receipts return `ErrCommandUnconfirmed`; closing an SDK exec handle does not prove that the guest process exited ([`command.go`](command.go)). Core uses `RunCommandCompute` only to run the exact wake command it authorizes: ```sh -oac-daemon resume --control-file /run/oac/daemon-suspend.json \ +oac-daemon resume --control-file CONTROL_FILE \ --environment-id ENVIRONMENT_ID --suspend-id SUSPENSION_ID ``` -The command checks the protected Environment and suspension identities and the parked daemon's PID and start time, then signals it through a Linux pidfd. The daemon reauthenticates and waits for Core's resume confirmation before it admits work. +`CONTROL_FILE` is the [suspend control file](../../../../docs/runtime-bootstrap.md#suspend-control). The command checks the protected Environment and suspension identities and the parked daemon's PID and start time, then signals it through a Linux pidfd. The daemon reauthenticates and waits for Core's resume confirmation before it admits work. ## Metrics diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index 4dd868c8e..21c294124 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -5,8 +5,10 @@ package main import ( "context" "encoding/json" + "path" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" sdk "github.com/superradcompany/microsandbox/sdk/go" @@ -17,7 +19,7 @@ import ( const bootstrapScript = ` import ctypes,json,os,stat,subprocess,sys b=json.load(sys.stdin) -for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages','/run/oac']: +for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages',sys.argv[1]]: os.makedirs(p,mode=0o700,exist_ok=True) if not stat.S_ISDIR(os.lstat(p).st_mode): raise RuntimeError('invalid bootstrap directory') os.chmod(p,0o700);os.chown(p,1000,1000) @@ -63,7 +65,6 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { sdk.WithLabels(labels), sdk.WithDetached(), sdk.WithQuietLogs(), sdk.WithNetwork(b.network()), sdk.WithEnv(map[string]string{ "HOME": "/home/runtime", "OAC_RUNTIME_HOME": "/home/runtime/.oac", - "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": "/run/oac/daemon-suspend.json", })) if e != nil { return wire.Response{}, e @@ -91,7 +92,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { } initialization, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() - result, e := runCommand(initialization, live, sandbox.Command{Args: []string{"/usr/bin/python3", "-I", "-S", "-c", bootstrapScript}, Stdin: data}, "0:0") + result, e := runCommand(initialization, live, sandbox.Command{Args: []string{"/usr/bin/python3", "-I", "-S", "-c", bootstrapScript, path.Dir(runtimebootstrap.SuspendControlFile)}, Stdin: data}, "0:0") if e != nil { return wire.Response{}, e }