From 4d9efc036721962c2852c38eca62c66ea4f91852 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Fri, 23 Oct 2026 20:48:16 +0000 Subject: [PATCH 01/12] Suspend and restore every deployment mode and pause E2B natively --- README.md | 2 +- README.zh-CN.md | 2 +- .../sandbox/SuspensionSettings.test.tsx | 2 + .../features/sandbox/SuspensionSettings.tsx | 4 +- apps/web/src/i18n/locales/zh-CN/sandbox.ts | 2 + .../agents-api/node-generation-protocol.md | 1 + contracts/agents-api/sandbox-deployment.md | 2 +- .../agents-api/zh/node-generation-protocol.md | 3 +- contracts/agents-api/zh/sandbox-deployment.md | 4 +- docs/sandbox-provider.md | 12 +- docs/zh/sandbox-provider.md | 12 +- internal/runtimebootstrap/bootstrap.go | 5 + .../server/managed_generation_operations.go | 11 ++ .../deploy/e2b/helper_contract_generated.py | 9 +- services/core/deploy/e2b/managed_init.py | 6 + services/core/deploy/e2b/managed_init_test.py | 5 +- services/core/internal/deployment/nodes.go | 3 +- .../provider_operations_fixture_test.go | 4 + .../internal/execution/runtime_compute.go | 22 ++- .../execution/runtime_compute_wake.go | 3 +- .../internal/execution/runtime_lifecycle.go | 5 +- .../execution/runtime_lost_compute_test.go | 3 + .../internal/sandbox/docker/operations.go | 4 + .../core/internal/sandbox/e2b/checkpoint.go | 180 ++++++++++++++++++ .../internal/sandbox/e2b/helper_contract.go | 14 +- .../sandbox/e2b/internal/contractgen/main.go | 17 +- .../core/internal/sandbox/e2b/operations.go | 46 +---- .../core/internal/sandbox/e2b/provider.go | 31 ++- .../sandbox/microsandbox/operations.go | 1 + .../internal/sandbox/microsandbox/provider.go | 5 + services/core/internal/sandbox/node/agent.go | 2 +- .../core/internal/sandbox/node/generations.go | 2 +- .../core/internal/sandbox/node/node_test.go | 2 +- .../core/internal/sandbox/node/operations.go | 1 + .../node/provider_operations_fixture_test.go | 5 + services/core/internal/sandbox/node/proxy.go | 4 + services/core/internal/sandbox/node/wire.go | 5 +- .../sandbox/providers/registration.go | 5 - .../sandbox/providers/registration_test.go | 8 +- .../internal/sandbox/providers/registry.go | 4 +- .../sandbox/providers/registry_test.go | 2 +- .../core/internal/sandbox/sandbox_provider.go | 14 +- .../provider_operations_fixture_test.go | 5 + .../runtime_compute_lifecycle_test.go | 44 ++++- .../runtime_wake_hint_integration_test.go | 5 + .../sandbox_deployment_view_test.go | 2 +- services/core/tools/e2b-provider/README.md | 6 +- .../e2b-provider/helper_contract_generated.py | 9 +- services/core/tools/e2b-provider/provider.py | 169 ++++++++++++++++ .../core/tools/e2b-provider/provider_test.py | 77 +++++++- .../tools/e2b-provider/testdata/contract.json | 63 +++--- .../tools/microsandbox-provider/bootstrap.go | 3 +- 52 files changed, 715 insertions(+), 142 deletions(-) create mode 100644 services/core/internal/sandbox/e2b/checkpoint.go diff --git a/README.md b/README.md index 070d59327..8601923f3 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ The OpenAI Agents API, open source and self-hosted. Run Codex, Claude Code or Mi - **Same API as OpenAI.** Set `OPENAI_BASE_URL` and the [official OpenAI SDK](https://developers.openai.com/api/docs/libraries), or plain HTTP, works unchanged. - **Your agent, your model.** Each Session runs Codex, Claude Code or MiniMax Code natively, as it runs on your machine, with the model provider you configure. See [Harness capabilities](https://openagentcore.dev/contracts/agents-api/harness-capabilities). -- **Compute that sleeps, workspaces that stay.** On microsandbox, which keeps every workspace on independent storage, idle sandboxes release compute and resume on a compatible node with their files intact. E2B, Docker and your own Linux, macOS or Windows machine also run Sessions. See [Sandbox Provider protocol](https://openagentcore.dev/docs/sandbox-provider). +- **Compute that sleeps, workspaces that stay.** On microsandbox, which keeps every workspace on independent storage, idle sandboxes release compute and resume on a compatible node with their files intact. E2B pauses idle sandboxes in place. Docker and your own Linux, macOS or Windows machine also run Sessions. See [Sandbox Provider protocol](https://openagentcore.dev/docs/sandbox-provider). - **Replaceable parts.** Sandbox Providers, workspace filesystems, Harnesses and model providers plug in through [defined protocols](AGENTS.md#protocols-at-every-boundary). ![Deployment overview](docs/assets/console-overview-en.webp) diff --git a/README.zh-CN.md b/README.zh-CN.md index e4cb85dbd..a92ff4679 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -17,7 +17,7 @@ - **与 OpenAI 相同的 API。** 设置 `OPENAI_BASE_URL` 后，[官方 OpenAI SDK](https://developers.openai.com/api/docs/libraries)或直接的 HTTP 调用无需改动即可使用。 - **你的 Agent，你的模型。** 每个 Session 原生运行 Codex、Claude Code 或 MiniMax Code，使用你配置的模型提供商，与在本机运行一致。详见 [Harness 能力](https://openagentcore.dev/zh/contracts/agents-api/harness-capabilities)。 -- **计算会休眠，工作区会保留。** 在 microsandbox 上（每个工作区都保存在独立存储中），空闲沙箱释放计算资源，并在兼容节点上带着文件恢复。E2B、Docker 以及你自己的 Linux、macOS 或 Windows 机器同样可以运行 Session。详见[沙箱提供商协议](https://openagentcore.dev/zh/docs/sandbox-provider)。 +- **计算会休眠，工作区会保留。** 在 microsandbox 上（每个工作区都保存在独立存储中），空闲沙箱释放计算资源，并在兼容节点上带着文件恢复。E2B 会就地暂停空闲沙箱。Docker 以及你自己的 Linux、macOS 或 Windows 机器同样可以运行 Session。详见[沙箱提供商协议](https://openagentcore.dev/zh/docs/sandbox-provider)。 - **部件可替换。** 沙箱提供商、工作区文件系统、Harness 和模型提供商都通过[既定协议](AGENTS.md#protocols-at-every-boundary)接入。 ![部署概览](docs/assets/console-overview-zh.webp) diff --git a/apps/web/src/features/sandbox/SuspensionSettings.test.tsx b/apps/web/src/features/sandbox/SuspensionSettings.test.tsx index db09a309f..f67a766e9 100644 --- a/apps/web/src/features/sandbox/SuspensionSettings.test.tsx +++ b/apps/web/src/features/sandbox/SuspensionSettings.test.tsx @@ -11,6 +11,8 @@ describe("suspension settings", () => { expect(html).toContain('id="sandbox-idle-seconds"'); expect(html).toContain('value="300"'); expect(html).toContain("24 hours"); + expect(html).toContain("Edit node"); + expect(renderToStaticMarkup( true} />)).not.toContain("Edit node"); expect(renderToStaticMarkup( true} />)).toBe(""); }); }); diff --git a/apps/web/src/features/sandbox/SuspensionSettings.tsx b/apps/web/src/features/sandbox/SuspensionSettings.tsx index 37404433e..99fb1f6fd 100644 --- a/apps/web/src/features/sandbox/SuspensionSettings.tsx +++ b/apps/web/src/features/sandbox/SuspensionSettings.tsx @@ -12,7 +12,7 @@ import { writeSandboxDeployment } from "./sandbox-deployment-write"; function RetentionHelp({ deployment }: { deployment: SandboxDeployment }) { const { t } = useTranslation("sandbox"); - return <>{t("Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume; capacity and a compatible node must be available.")}{deployment.specification?.workspace ? <> {t("Snapshot expiry releases memory state. With independent workspace storage, files and conversation remain available for a cold continuation; processes restart.")} : null}; + return <>{t(deployment.mode === "direct" ? "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume." : "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume; capacity and a compatible node must be available.")}{deployment.specification?.workspace ? <> {t("Snapshot expiry releases memory state. With independent workspace storage, files and conversation remain available for a cold continuation; processes restart.")} : null}; } /** Settings edits the deployment-wide idle time; each node's limits are on Nodes. */ @@ -41,7 +41,7 @@ export function SuspensionForm({ deployment, disabled, onSave }: { deployment: S if (!policy) return null; const idle = draft ?? String(policy.idle_seconds); const valid = /^\d+$/.test(idle) && Number(idle) >= 30 && Number(idle) <= 86400; - return
+ return
{ event.preventDefault(); if (valid && !disabled && await onSave(Number(idle))) setDraft(null); }}>
{t("Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged. Capacity pressure may suspend an idle sandbox after 15 seconds.")}
diff --git a/apps/web/src/i18n/locales/zh-CN/sandbox.ts b/apps/web/src/i18n/locales/zh-CN/sandbox.ts index 7f2768737..8feac385c 100644 --- a/apps/web/src/i18n/locales/zh-CN/sandbox.ts +++ b/apps/web/src/i18n/locales/zh-CN/sandbox.ts @@ -8,6 +8,8 @@ export const sandbox = { "Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged. Capacity pressure may suspend an idle sandbox after 15 seconds.": "整个部署统一生效：30–86,400 秒。适用于现有和新建的运行中沙箱，无需重启。已经开始的挂起和快照保留期不变。容量紧张时，沙箱空闲 15 秒后可能提前挂起。", "Enter a whole number from 30 to 86,400.": "请输入 30 到 86,400 之间的整数。", "Automatic suspension": "自动挂起", + "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume.": "这是新挂起任务采用的当前策略，不是现有快照的到期倒计时。下一轮 Turn 会请求恢复。", + "The idle policy applies to the whole deployment.": "空闲策略对整个部署生效。", "Snapshot retention": "快照保留期", "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume; capacity and a compatible node must be available.": "这是新挂起任务采用的当前策略，不是现有快照的到期倒计时。下一轮 Turn 会请求恢复，需要可用容量及兼容节点。", "Snapshot expiry releases memory state. With independent workspace storage, files and conversation remain available for a cold continuation; processes restart.": "快照到期后释放内存状态。使用独立工作区存储时，文件和对话仍可用于冷启动续接，进程会重新启动。", diff --git a/contracts/agents-api/node-generation-protocol.md b/contracts/agents-api/node-generation-protocol.md index c6b2972b1..a7a7e997a 100644 --- a/contracts/agents-api/node-generation-protocol.md +++ b/contracts/agents-api/node-generation-protocol.md @@ -79,6 +79,7 @@ Each operation carries its own arguments and returns the following result on suc | `initial` | `Initial` | None | `compute` | | `new_compute` | `NewCompute` | Positive compute `generation` and optional `snapshot` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | None | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` and `command` | `command` | diff --git a/contracts/agents-api/sandbox-deployment.md b/contracts/agents-api/sandbox-deployment.md index 50f0307af..1628bc56f 100644 --- a/contracts/agents-api/sandbox-deployment.md +++ b/contracts/agents-api/sandbox-deployment.md @@ -220,7 +220,7 @@ Some fields keep one name across providers but differ in meaning, or do not appl | Independent workspace filesystem | Required: [Kernel NFS adapter](../../docs/workspace-provider.md#kernel-nfs-adapter), configured separately from compute | Unsupported | Unsupported | | Deployment `specification.resources` | `cpus`, `memory_mib`, `root_disk_mib` and, as a quota request, `environment_disk_mib` | `cpus` and `memory_mib`, equal to the ready template build's and taken from it when omitted; no disk fields | `cpus` and `memory_mib`; no disk quota | | Deployment `metadata.template_build` | Absent: `metadata` is empty | The build as Core read it when the selection was saved | Absent: `metadata` is empty | -| Deployment `suspension` | `{idle_seconds, retention_seconds}` | `null`; Core does not suspend E2B sandboxes | `null` | +| Deployment `suspension` | `{idle_seconds, retention_seconds}` | `{idle_seconds, retention_seconds}`; Core pauses an idle E2B sandbox with the SDK's native pause | `null` | | Deployment `resources.allocations`, `resources.pending` | Totals across all nodes | Core's unreleased E2B sandboxes, and hosted Environments waiting for one | Totals across all nodes | | Enrollment-token `max_active`, `max_retained` | Both limits apply; an omitted limit is derived at enrollment | 409 `sandbox_deployment_conflict`, after the 400 capacity checks; E2B has no nodes | `max_retained` always equals `max_active` | | Node list and detail | Enrolled nodes | Empty list; detail returns 404 | Enrolled nodes | diff --git a/contracts/agents-api/zh/node-generation-protocol.md b/contracts/agents-api/zh/node-generation-protocol.md index ac649c21d..64dd5a810 100644 --- a/contracts/agents-api/zh/node-generation-protocol.md +++ b/contracts/agents-api/zh/node-generation-protocol.md @@ -1,7 +1,7 @@ --- title: "沙箱节点协议" source: contracts/agents-api/node-generation-protocol.md -source_hash: 090803f51d2e29d8900cfe8612b0eb21363a0fd2687daf3d368cc746dd65809f +source_hash: 3648a80013f1ae90031430c5bdf28d6ee5b59f55dda4b168dfeb99ab5bcec3e2 --- 沙箱节点协议连接 Core 与 Docker 或 microsandbox 主机：主机针对其本地提供商执行提供商操作，并报告就绪状态、主机测量值及其持有的部署代次。 @@ -81,6 +81,7 @@ Core 发送包含以下内容的 `request` 帧： | `initial` | `Initial` | 无 | `compute` | | `new_compute` | `NewCompute` | 大于零的计算 `generation` 和可选的 `snapshot` | `compute` | | `compute` | `GetCompute` | `compute` | `state` | +| `renew_compute` | `RenewCompute` | `compute` | `state` | | `kill_compute` | `KillCompute` | `compute` | 无 | | `resume_compute` | `ResumeCompute` | `compute` | `state` | | `command_compute` | `RunCommandCompute` | `compute` 和 `command` | `command` | diff --git a/contracts/agents-api/zh/sandbox-deployment.md b/contracts/agents-api/zh/sandbox-deployment.md index 306cefcb9..5b13320e1 100644 --- a/contracts/agents-api/zh/sandbox-deployment.md +++ b/contracts/agents-api/zh/sandbox-deployment.md @@ -1,7 +1,7 @@ --- title: "沙箱部署" source: contracts/agents-api/sandbox-deployment.md -source_hash: 4b77c11fe9a80e4b6d200ca161ab53e006bfc46b185262ee858fc65d75b5112a +source_hash: 0ceb33096c17cf3ba1c504bc4628ba0c281358849819f965accfeb6cbd5f2d47 --- 沙箱部署为 Core 管理的 `openai_hosted` 执行选择沙箱提供商和每个沙箱的资源。PostgreSQL 为每个安装维护一个当前有效选择；Web 和 Core API 写入同一配置。节点文件保存其已安装副本和特定于主机的路径，且不能覆盖其资源。Runtime 不属于该选择：每个节点运行其所属安装验证过的 Runtime。该选择独立于 Harness。部署可以保持未配置状态，没有节点；此时它拒绝托管准入。 @@ -225,7 +225,7 @@ Core 用一条规则，从节点主机和单个沙箱的大小推导推荐值。 | 独立工作区文件系统 | 必需：[Kernel NFS adapter](../../../docs/zh/workspace-provider.md#kernel-nfs-adapter)，与计算资源分开配置 | 不支持 | 不支持 | | 部署 `specification.resources` | `cpus`、`memory_mib`、`root_disk_mib`，以及作为配额请求的 `environment_disk_mib` | `cpus` 和 `memory_mib`，必须等于就绪模板构建中的值；省略时取自该构建；无磁盘字段 | `cpus` 和 `memory_mib`；无磁盘配额 | | 部署 `metadata.template_build` | 不存在：`metadata` 为空 | Core 保存选择时读取到的构建 | 不存在：`metadata` 为空 | -| 部署 `suspension` | `{idle_seconds, retention_seconds}` | `null`；Core 不暂停 E2B 沙箱 | `null` | +| 部署 `suspension` | `{idle_seconds, retention_seconds}` | `{idle_seconds, retention_seconds}`；Core 用 SDK 的原生 pause 暂停空闲的 E2B 沙箱 | `null` | | 部署 `resources.allocations`、`resources.pending` | 所有节点的总数 | Core 中尚未释放的 E2B 沙箱，以及正在等待沙箱的托管 Environment | 所有节点的总数 | | 注册令牌 `max_active`、`max_retained` | 两个限制均适用；省略的限制在注册时推导 | 先执行 400 容量检查，然后返回 409 `sandbox_deployment_conflict`；E2B 没有节点 | `max_retained` 始终等于 `max_active` | | 节点列表和详情 | 已注册节点 | 空列表；详情返回 404 | 已注册节点 | diff --git a/docs/sandbox-provider.md b/docs/sandbox-provider.md index ca101398a..d62b53c21 100644 --- a/docs/sandbox-provider.md +++ b/docs/sandbox-provider.md @@ -62,10 +62,10 @@ Every provider returns a complete `ProviderOperations()` declaration with one en | --- | --- | --- | | `Create`, `GetInfo`, `Renew`, `Kill`, `RunCommand` | Supported | Allocation lifecycle and bounded commands | | `Observe` | Explicit decision | Ownership-checked read-only observation of one allocation | -| `Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method; supported only by a `nodes` registration | Exact compute incarnations, capture and restore, retained-source resume and cleanup | +| `Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute` | The same decision for every checkpoint method | Exact compute incarnations, capture and restore, retained-source resume and cleanup | | `RestorePreparation`, `SavePreparation` | The same decision for both | Reuse of a finished Environment preparation | -`Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). +`Initial` and `NewCompute` construct compute references without allocating, `ResumeCompute` thaws only the same resident instance after an aborted pause, `RenewCompute` extends the native lease of the exact running incarnation where the Provider has one and otherwise observes exactly like `GetCompute`, and `RunCommandCompute` runs a bounded command in one exact compute incarnation. Core uses `RunCommandCompute` to wake a parked daemon after a restore ([`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)). `SavePreparation(reference, key)` stores what preparation wrote under `/environment` (workspace, packages and initialization, never Runtime state) in the allocation's initial compute after a complete preparation, and `RestorePreparation(reference, key)` puts a stored one back and returns `false` when it holds no valid entry. The `key` is opaque: Core derives it from the Project and everything that reaches preparation, never from the Environment, Session, Harness or Agent, and the Provider never interprets it. The Provider owns storage, scoping to its Runtime image, expiry and eviction. An entry is valid for 24 hours after its save, and a restore must not use an expired one. Core calls `RestorePreparation` after the Runtime connects and, on a hit, sends only the `finalize` operation before completing initialization. On a miss it prepares in full and calls `SavePreparation`; a save error is logged and never fails the Session. A Provider that declares neither always prepares in full. The microsandbox adapter keeps one archive per key in a private directory under the Runtime home. @@ -109,7 +109,7 @@ Every call receives a bounded context. Expiry or cancellation ends the caller's `ErrInvalid`, `ErrOwnership`, `ErrExists`, `ErrNotFound`, `ErrComputeUnconfirmed`, `ErrCommandUnconfirmed` and, for `Resume` only, `ErrCheckpointUnusable` keep their defined meanings. An unclassified native or transport error is unknown, never permission to retry a mutation. Core never reads provider diagnostics as lifecycle truth or exposes native error text or credentials; the node transport maps errors to fixed codes, and direct SDK details stay private. -Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. `GetCompute` returns `ErrNotFound` only when the owning node has verified that the exact incarnation no longer exists; an instance with the same name and another native ID is `ErrOwnership`, and an unreachable node or backend is an unknown result. A `ComputeState.Status` of `stopped` or `crashed` means the adapter verified that the exact incarnation’s process no longer exists; its record or disks can remain until `KillCompute` removes them. `paused`, `running` without a Runtime connection, and unknown states do not establish termination. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. +Checkpoint support adds `Compute` generation, name and ID and `SnapshotIdentity`; persist operation IDs and the provider's snapshot provenance unchanged. `ObserveOnly` on suspend or resume observes the previous attempt and never starts another capture or restore. `ResumeCompute` only thaws the retained source and never cold-starts a stopped one. `RenewCompute` never revives stopped compute, and Core calls it instead of `GetCompute` while a running allocation's Runtime is connected, because `Renew` addresses only the initial incarnation. Cleanup targets the exact compute incarnation and snapshot, not whatever instance now has the same name. `GetCompute` returns `ErrNotFound` only when the owning node has verified that the exact incarnation no longer exists; an instance with the same name and another native ID is `ErrOwnership`, and an unreachable node or backend is an unknown result. A `ComputeState.Status` of `stopped` or `crashed` means the adapter verified that the exact incarnation’s process no longer exists; its record or disks can remain until `KillCompute` removes them. `paused`, `running` without a Runtime connection, and unknown states do not establish termination. Read [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) and its failure tests before declaring checkpoint support. ### Four distinct readiness facts @@ -144,7 +144,7 @@ A node configuration, `sandbox.NodeConfig`, holds `provider`, `generation`, `ins - A `nodes` registration has only `BuildLocal`, and a `direct` registration only `BuildDirect`; missing, mixed or unknown modes are rejected. - The configuration adapter and a complete operation declaration are mandatory, so an incomplete registration cannot publish a partial projection. A declared default size must pass `ValidatePolicy("", Policy)`. -- Checkpoint is admitted only for a `nodes` registration, because the common lifecycle suspends only node allocations; registration rejects a `direct` Provider that declares it. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. +- Checkpoint is admitted for both modes: the common lifecycle suspends, retains, restores and cleans up a `direct` allocation exactly as a node allocation, with no node. A registration carries no suspension values: Core applies its one [suspension policy](#suspension) to every Provider that declares checkpoint support. The configuration adapter must be non-nil, including its concrete value. Every `ConfigurationRequirements` field needs an explicit valid decision: `Credential` and `PublicOrigin` are `Required` or `NotRequired`, and `Discovery`, `SelectionDiscovery` and `CredentialVerification` use the shared supported or unsupported declaration with a safe reason. A new requirement field needs an explicit validation update and never inherits an existing decision. Requiring a credential does not promise the `VerifyCredential` operation. These checks establish complete registration, not correct native SDK behavior; constructor and adapter contract tests still apply. @@ -193,7 +193,7 @@ An allocation moves to `cleanup_pending` when its Session is deleted or archived ## Checkpoint lifecycle -A Provider that declares the checkpoint operations (`Initial`, `NewCompute`, `GetCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute`) lets Core suspend an idle allocation to a snapshot and restore it on the same or another compatible node. Core records the allocation's `compute_phase`; it is `disabled` for a Provider that does not declare checkpoint support, and the other phases move as follows. Deletion, archive, reset and retention expiry release the allocation from any phase after the cleanup described in [What Core guarantees to a Provider](#what-core-guarantees-to-a-provider). +A Provider that declares the checkpoint operations (`Initial`, `NewCompute`, `GetCompute`, `RenewCompute`, `Suspend`, `Resume`, `ResumeCompute`, `KillCompute`, `DeleteSnapshot`, `RunCommandCompute`) lets Core suspend an idle allocation to a snapshot and restore it on the same or another compatible node. Core records the allocation's `compute_phase`; it is `disabled` for a Provider that does not declare checkpoint support, and the other phases move as follows. Deletion, archive, reset and retention expiry release the allocation from any phase after the cleanup described in [What Core guarantees to a Provider](#what-core-guarantees-to-a-provider). ```mermaid stateDiagram-v2 @@ -260,6 +260,8 @@ A checkpoint-capable generation reports `CheckpointCompatibility`: opaque `Artif Recovery uses `Resume.ObserveOnly` for the persisted target and operation. `RestoreAttemptClosed` is an exact operation ID, not a generic absence flag: the adapter may return it only after durably closing an attempt that never entered native execution and fencing every late request for that ID. An adapter may also close a durably admitted attempt when it can prove that its current invocation failed before dispatching native restoration. The microsandbox adapter does this for the fresh request that created the admission, on any failure before it dispatches native restoration. An expired wire deadline or an unclassified failure closes the attempt, and Core then persists a new operation ID. A snapshot that is missing, fails identity or integrity verification, or fails the adapter's qualification for this generation, such as a resource, workspace or runtime image mismatch, is also closed and reported as `ErrCheckpointUnusable`. That outcome is durable: `Resume.ObserveOnly` for the same operation reports it again, and late requests for the operation ID stay fenced. A Provider never returns it once native restoration was dispatched. On `ErrCheckpointUnusable` Core never restores that snapshot again and [ends its retention at once](#cold-replacement-after-checkpoint-retention). Once native restoration has been dispatched, an unknown outcome retains the same target, ownership and original retention deadline; a missing native listing, helper death or timeout does not authorize replay. Such a restore can remain unavailable until explicit deletion or ordinary retention expiry, and pending inputs can reach their public deadline while waiting. User deletion still requires exact native settlement before resources and ownership can be released. +The E2B adapter checkpoints with the SDK's native pause and `connect`. One sandbox spans every generation of an allocation: `Suspend` pauses it with a memory snapshot, so the daemon survives, and `Resume` reconnects to the same sandbox. The helper's private receipt records the current generation, so a superseded generation is `ErrNotFound`, and `KillCompute` and `DeleteSnapshot` never touch a sandbox that another generation owns. `CheckpointCompatibility` names the endpoint and installation and the template build, and a snapshot that does not match is `ErrCheckpointUnusable` before any native call. E2B keeps no independent workspace filesystem, so retention expiry follows the unqualified branch below. + The microsandbox adapter requires an explicit private `checkpoint_root`, outside every guest-accessible filesystem. Its directory is mode `0700`; a private namespace marker identifies the actual checkpoint store. The default installation creates a node-local private store. Cross-node restoration requires operators to mount the same private store on each participating node; mount paths may differ. Each retained checkpoint is an uncompressed native snapshot of the guest's memory and root-disk layers, so a shared store carries that volume for every suspension and cross-node restore. Runtime state directories and workspace bindings are never used to infer this root. The adapter verifies the checkpoint, external filesystem identity and native execution class again before restoration. Restored RAM and file descriptors do not promise continuity of external TCP connections or replay safety for application side effects. ### Cold replacement after checkpoint retention diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index abcb0c8cb..67427f47c 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -64,10 +64,10 @@ Docker 等没有原生可续期租约的 backend 仍遵守 Core 的 hosted expir | --- | --- | --- | | `Create`、`GetInfo`、`Renew`、`Kill`、`RunCommand` | 支持 | Allocation 生命周期与有界命令 | | `Observe` | 明确决定 | 检查所有权、只读地观测一个 allocation | -| `Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致；仅 `nodes` 注册可以支持 | 精确计算实例、捕获与恢复、保留源恢复和清理 | +| `Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute` | 所有 checkpoint 方法决定一致 | 精确计算实例、捕获与恢复、保留源恢复和清理 | | `RestorePreparation`、`SavePreparation` | 两者决定一致 | 复用已完成的 Environment 准备结果 | -`Initial` 和 `NewCompute` 构造 compute reference，不分配资源；`ResumeCompute` 在暂停中止后仅解冻同一驻留实例；`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon（[`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)）。 +`Initial` 和 `NewCompute` 构造 compute reference，不分配资源；`ResumeCompute` 在暂停中止后仅解冻同一驻留实例；`RenewCompute` 在提供商有原生租约时延长精确运行实例的租约，否则与 `GetCompute` 完全一样只做观察；`RunCommandCompute` 在一个精确 compute incarnation 中运行有界命令。Core 使用 `RunCommandCompute` 在恢复后唤醒 parked daemon（[`runtime_compute_wake.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute_wake.go)）。 `SavePreparation(reference, key)` 在准备完整完成后保存该分配初始 compute 中准备写入 `/environment` 的内容（workspace、packages 和 initialization，从不包括 Runtime 状态），`RestorePreparation(reference, key)` 把已保存的内容放回，没有有效条目时返回 `false`。`key` 是不透明的：Core 由 Project 和所有进入准备的内容推导它，从不使用 Environment、Session、Harness 或 Agent，提供商也从不解释它。存储、按 Runtime 镜像划分范围、过期和淘汰由提供商负责。条目在保存后 24 小时内有效，恢复不得使用已过期的条目。Core 在 Runtime 连接后调用 `RestorePreparation`，命中时只发送 `finalize` 操作即完成初始化。未命中时完整准备并调用 `SavePreparation`；保存错误只记录日志，不会使 Session 失败。两者都未声明的提供商始终完整准备。microsandbox 适配器在 Runtime home 下的私有目录中为每个 key 保存一个归档。 @@ -111,7 +111,7 @@ Microsandbox 在返回创建不存在回执前永久关闭初始 Create 准入 `ErrInvalid`、`ErrOwnership`、`ErrExists`、`ErrNotFound`、`ErrComputeUnconfirmed`、`ErrCommandUnconfirmed`，以及仅用于 `Resume` 的 `ErrCheckpointUnusable` 保持其定义含义。未分类原生或 transport error 表示未知，不授权重试 mutation。Core 不将提供商 diagnostics 读作生命周期事实，也不暴露原生错误文本或凭据；node transport 将错误映射为固定 code，直接 SDK 细节保持私有。 -Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`；原样持久化 operation ID 与提供商 snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试，不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源，不冷启动已停止源。清理针对精确 compute incarnation 和 snapshot，不针对当前同名实例。只有所属节点已验证精确 incarnation 不再存在时，`GetCompute` 才返回 `ErrNotFound`；同名但 native ID 不同的实例为 `ErrOwnership`，节点或提供商不可达属于未知结果。`ComputeState.Status` 为 `stopped` 或 `crashed` 表示 adapter 已验证该精确 incarnation 的进程不再存在；其记录或磁盘可保留到 `KillCompute` 将其移除。`paused`、没有 Runtime 连接的 `running` 和未知状态均不证明进程已终止。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 +Checkpoint 支持增加 `Compute` generation、name、ID 和 `SnapshotIdentity`；原样持久化 operation ID 与提供商 snapshot provenance。suspend 或 resume 的 `ObserveOnly` 仅观察上次尝试，不启动另一 capture 或 restore。`ResumeCompute` 仅解冻保留源，不冷启动已停止源。`RenewCompute` 从不复活已停止的 compute；运行中 allocation 的 Runtime 保持连接时，Core 调用它而不是 `GetCompute`，因为 `Renew` 只寻址初始 incarnation。清理针对精确 compute incarnation 和 snapshot，不针对当前同名实例。只有所属节点已验证精确 incarnation 不再存在时，`GetCompute` 才返回 `ErrNotFound`；同名但 native ID 不同的实例为 `ErrOwnership`，节点或提供商不可达属于未知结果。`ComputeState.Status` 为 `stopped` 或 `crashed` 表示 adapter 已验证该精确 incarnation 的进程不再存在；其记录或磁盘可保留到 `KillCompute` 将其移除。`paused`、没有 Runtime 连接的 `running` 和未知状态均不证明进程已终止。声明 checkpoint 支持前阅读 [`runtime_compute.go`](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/services/core/internal/execution/runtime_compute.go) 及其失败测试。 ### 四个独立就绪事实 {#four-distinct-readiness-facts} @@ -146,7 +146,7 @@ node 配置 `sandbox.NodeConfig` 包含 `provider`、`generation`、`installatio - `nodes` 注册仅有 `BuildLocal`，`direct` 注册仅有 `BuildDirect`；缺失、混合或未知 mode 被拒绝。 - configuration adapter 与完整 operation 声明都是必需项，因此不完整注册不能发布部分投影。声明的默认大小必须通过 `ValidatePolicy("", Policy)`。 -- Checkpoint 仅准入 `nodes` 注册，因为公共 lifecycle 只暂停 node allocation；声明 checkpoint 的 `direct` 提供商会被注册拒绝。注册不携带 suspension 数值：Core 对每个声明 checkpoint 支持的提供商应用同一个 [suspension policy](#suspension)。 +- Checkpoint 对两种模式都准入：公共 lifecycle 对 `direct` allocation 的暂停、保留、恢复和清理与 node allocation 完全一致，只是没有 node。注册不携带 suspension 数值：Core 对每个声明 checkpoint 支持的提供商应用同一个 [suspension policy](#suspension)。 configuration adapter 必须非 nil，包括其具体值。每个 `ConfigurationRequirements` 字段都需要明确有效的决定：`Credential` 和 `PublicOrigin` 为 `Required` 或 `NotRequired`，`Discovery`、`SelectionDiscovery` 和 `CredentialVerification` 使用共享 supported 或 unsupported 声明并携带安全 reason。新增 requirement field 需要明确更新验证，不继承已有决定。要求凭据不承诺支持 `VerifyCredential` 操作。这些检查证明注册完整，不证明原生 SDK 行为正确；constructor 和 adapter 契约测试仍然适用。 @@ -195,7 +195,7 @@ stateDiagram-v2 ## 检查点生命周期 {#checkpoint-lifecycle} -声明了 checkpoint 操作（`Initial`、`NewCompute`、`GetCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute`）的提供商，使 Core 能把空闲 allocation 暂停为快照，并在同一或另一个兼容 node 上恢复。Core 记录 allocation 的 `compute_phase`；未声明 checkpoint 支持的提供商为 `disabled`，其余阶段按下图变化。删除、归档、重置和保留期到期会在完成[Core 对提供商的保证](#what-core-guarantees-to-a-provider)中描述的清理后，从任意阶段释放 allocation。 +声明了 checkpoint 操作（`Initial`、`NewCompute`、`GetCompute`、`RenewCompute`、`Suspend`、`Resume`、`ResumeCompute`、`KillCompute`、`DeleteSnapshot`、`RunCommandCompute`）的提供商，使 Core 能把空闲 allocation 暂停为快照，并在同一或另一个兼容 node 上恢复。Core 记录 allocation 的 `compute_phase`；未声明 checkpoint 支持的提供商为 `disabled`，其余阶段按下图变化。删除、归档、重置和保留期到期会在完成[Core 对提供商的保证](#what-core-guarantees-to-a-provider)中描述的清理后，从任意阶段释放 allocation。 ```mermaid stateDiagram-v2 @@ -262,6 +262,8 @@ Core 在初始化完成、没有 root 或 Subagent Turn 处于排队、进行中 恢复用 `Resume.ObserveOnly` 观察已持久化的 target 与 operation。`RestoreAttemptClosed` 是精确 operation ID，不是通用 absence 标志：adapter 只有在持久关闭从未进入原生执行的 attempt，并隔离该 ID 的所有迟到请求后才能返回。如果 adapter 能证明当前调用在派发原生恢复之前失败，也可以关闭已持久 admitted 的 attempt。microsandbox adapter 对创建该 admission 的 fresh 请求，在派发原生恢复之前的任何失败上执行这种关闭。wire deadline 到期或未分类失败会关闭该 attempt，Core 随后持久化新的 operation ID。快照缺失、未通过身份或完整性校验，或未通过 adapter 对当前 generation 的资格检查（例如资源、workspace 或 runtime image 不匹配）时，也会关闭该 attempt，并报告为 `ErrCheckpointUnusable`。该结果是持久的：同一 operation 的 `Resume.ObserveOnly` 会再次报告它，该 operation ID 的迟到请求仍被隔离。一旦已派发原生恢复，提供商不会返回它。收到 `ErrCheckpointUnusable` 后，Core 不再恢复该快照，并[立即结束其保留](#cold-replacement-after-checkpoint-retention)。一旦已派发原生恢复，结果未知的 attempt 保留相同 target、所有权与原始 retention deadline；原生列表缺失、helper 死亡或超时都不授权重放。这种恢复可能一直不可用，直到显式删除或正常保留期到期；等待中的 input 可能达到其公共 deadline。用户删除仍需精确原生 settlement，之后才能释放资源与所有权。 +E2B adapter 使用 SDK 的原生 pause 与 `connect` 做 checkpoint。一个 sandbox 跨越 allocation 的所有 generation：`Suspend` 以内存快照暂停它，daemon 因此存活，`Resume` 重新连接同一个 sandbox。helper 的私有 receipt 记录当前 generation，所以被取代的 generation 返回 `ErrNotFound`，`KillCompute` 和 `DeleteSnapshot` 从不触碰由其他 generation 拥有的 sandbox。`CheckpointCompatibility` 指明 endpoint、安装和模板构建，不匹配的快照在任何原生调用之前即为 `ErrCheckpointUnusable`。E2B 没有独立的 workspace 文件系统，因此保留期到期走下面的未满足条件分支。 + microsandbox adapter 要求显式的私有 `checkpoint_root`，位于所有 guest 可访问文件系统之外。目录权限为 `0700`，私有 namespace marker 标识真实检查点存储。默认安装创建 node 本地私有 store。跨 node 恢复要求运维为参与 node 挂载同一个私有 store，挂载路径可以不同。每个保留的检查点都是 guest 内存与根磁盘层的未压缩原生快照，因此共享 store 要为每次挂起和跨 node 恢复承载这部分数据量。Runtime state 目录与 workspace binding 都不用于推导该 root。adapter 在恢复前再次验证检查点、外部文件系统身份和原生 execution class。恢复 RAM 与文件描述符不保证外部 TCP 连接连续，也不保证应用副作用可安全重放。 ### 检查点保留期后的冷替换 {#cold-replacement-after-checkpoint-retention} diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index 5cbb87ff9..54d069832 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -25,6 +25,11 @@ const PublicWorkspace = "/workspace" // RuntimeWorkspace is the workspace directory of the packaged Runtime layout. const RuntimeWorkspace = "/environment/workspace" +// SuspendControlFile is where the Runtime daemon publishes its suspend control. +// Its directory is private to the Runtime user. Providers enable the control by +// setting OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE to it, and Core's wake command reads it. +const SuspendControlFile = "/run/oac/daemon-suspend.json" + var ErrInvalid = errors.New("invalid Runtime bootstrap input") // Connection is a current-version launch input, not Runtime's private auth store. diff --git a/services/core/cmd/server/managed_generation_operations.go b/services/core/cmd/server/managed_generation_operations.go index bacf142d9..085942b2a 100644 --- a/services/core/cmd/server/managed_generation_operations.go +++ b/services/core/cmd/server/managed_generation_operations.go @@ -105,6 +105,17 @@ func (p *generationRouter) ResumeCompute(ctx context.Context, r sandbox.Referenc defer done() return v.ResumeCompute(ctx, r, c) } +func (p *generationRouter) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + if err := providercontract.Require(p, "RenewCompute"); err != nil { + return sandbox.ComputeState{}, err + } + v, done, err := p.route(ctx, r) + if err != nil { + return sandbox.ComputeState{}, err + } + defer done() + return v.RenewCompute(ctx, r, c) +} func (p *generationRouter) RestorePreparation(ctx context.Context, r sandbox.Reference, key string) (bool, error) { if err := providercontract.Require(p, "RestorePreparation"); err != nil { return false, err diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index ee81a1327..bf7f1aa4a 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,6 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" -ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] +ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] @@ -9,9 +9,10 @@ MAX_OUTPUT = 1048576 MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute","renew_compute","suspend","resume","kill_compute","delete_snapshot","command_compute","resume_compute"] PROTOCOL_VERSION = 1 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Snapshot","Deadline"] +RESPONSE_FIELDS = ["Version","Info","Command","State","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index 3cd2e11cc..7b65fa505 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -44,6 +44,12 @@ def initialize(): binding = identity(payload) shared.write_private(root / 'managed-launch.json', binding) environment = shared.prepare_runtime() + # The daemon publishes its suspend control here; Core's wake command reads it. + control = Path(contract.SUSPEND_CONTROL_FILE) + control.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + os.chown(control.parent, 1000, 1000) + control.parent.chmod(0o700) + environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'] = str(control) connection = Path(environment['OAC_RUNTIME_HOME']).parent / 'runtime-bootstrap.json' shared.write_private(connection, payload['RuntimeBootstrap'], owner=1000) source.unlink() diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index ae6118a2c..5cc9cce76 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -63,7 +63,9 @@ def exercise(self, failed=False): image_env = {'PATH': '/usr/local/bin:/usr/bin:/bin', 'OAC_RUNTIME_HOME': str(Path(temporary) / '.oac')} with patch.object(managed_init.shared, 'ROOT', root), patch.object(managed_init.shared, 'PROFILE', profile), \ patch.object(managed_init.shared, 'prepare_runtime', return_value=image_env), \ - patch.object(managed_init.os, 'fchown'), patch.object(managed_init.subprocess, 'Popen', process): + patch.object(managed_init.os, 'fchown'), patch.object(managed_init.os, 'chown'), \ + patch.object(managed_init.contract, 'SUSPEND_CONTROL_FILE', str(Path(temporary) / 'run' / 'daemon-suspend.json')), \ + patch.object(managed_init.subprocess, 'Popen', process): if failed: with self.assertRaises(RuntimeError): managed_init.initialize() @@ -80,6 +82,7 @@ def exercise(self, failed=False): self.assertNotIn(data['RuntimeBootstrap']['credential'], json.dumps(process.call_args.kwargs['env'])) self.assertEqual(json.loads(connection_file.read_text())['environment_id'], data['EnvironmentID']) self.assertEqual(process.call_args.kwargs['user'], 1000) + self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], str(Path(temporary) / 'run' / 'daemon-suspend.json')) if failed: self.assertFalse((root / 'managed-ready.json').exists()) else: diff --git a/services/core/internal/deployment/nodes.go b/services/core/internal/deployment/nodes.go index 1ad434a86..2b0ab60cd 100644 --- a/services/core/internal/deployment/nodes.go +++ b/services/core/internal/deployment/nodes.go @@ -13,7 +13,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/coremetrics" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" "github.com/google/uuid" ) @@ -605,7 +604,7 @@ func (s *Service) recordGenerations(tx NodeTx, d Record, n StoredNode, statuses if err != nil { return err } - checkpointSupported := adapter.Operations()["Initial"].State == providercontract.Supported + checkpointSupported := sandbox.SupportsCheckpoint(adapter.Operations()) seen := map[uint64]bool{} for _, status := range statuses { if status.Checkpoint != nil && (status.State != "ready" || !checkpointSupported || status.Checkpoint.Validate() != nil) || status.State == "ready" && checkpointSupported && status.Checkpoint == nil { diff --git a/services/core/internal/execution/provider_operations_fixture_test.go b/services/core/internal/execution/provider_operations_fixture_test.go index 58b17c3a8..44acb244f 100644 --- a/services/core/internal/execution/provider_operations_fixture_test.go +++ b/services/core/internal/execution/provider_operations_fixture_test.go @@ -24,6 +24,7 @@ func (*lifecycleOnlySandbox) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -56,6 +57,9 @@ func (*lifecycleOnlySandbox) RunCommandCompute(context.Context, sandbox.Referenc func (*lifecycleOnlySandbox) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: "fixture_operation_not_supported"} } +func (*lifecycleOnlySandbox) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} func (*lifecycleOnlySandbox) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } diff --git a/services/core/internal/execution/runtime_compute.go b/services/core/internal/execution/runtime_compute.go index f3bc4f863..b25ee9ac7 100644 --- a/services/core/internal/execution/runtime_compute.go +++ b/services/core/internal/execution/runtime_compute.go @@ -60,8 +60,8 @@ func logComputeStep(ctx context.Context, owner deployment.Allocation, step strin func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.Allocation) (err error) { defer func() { err = withObservationOwner(owner, err) }() p := r.config.Provider - if err := providercontract.Require(p, "Initial"); err != nil { - return err + if !sandbox.SupportsCheckpoint(p.ProviderOperations()) { + return providercontract.Require(p, "Initial") } initial, err := p.Initial(ctx, runtimeReference(owner)) if err != nil { @@ -81,8 +81,8 @@ func (r *runtimeLifecycle) enableCompute(ctx context.Context, owner deployment.A func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment.Allocation) (err error) { defer func() { err = withObservationOwner(owner, err) }() p := r.config.Provider - if err := providercontract.Require(p, "Initial"); err != nil { - return err + if !sandbox.SupportsCheckpoint(p.ProviderOperations()) { + return providercontract.Require(p, "Initial") } var state runtimeCompute if json.Unmarshal(owner.ComputeState, &state) != nil || state.Current.ID == "" { @@ -121,7 +121,14 @@ func (r *runtimeLifecycle) observeCompute(ctx context.Context, owner deployment. func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxProvider, owner deployment.Allocation, state runtimeCompute) (err error) { defer func() { err = withObservationOwner(owner, err) }() - compute, err := p.GetCompute(ctx, runtimeReference(owner), state.Current) + // With an open Runtime connection the observation also extends the native + // lease, as the disabled path does; otherwise it only observes. + peer, peerErr := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) + observe := p.GetCompute + if peerErr == nil { + observe = p.RenewCompute + } + compute, err := observe(ctx, runtimeReference(owner), state.Current) if errors.Is(err, sandbox.ErrNotFound) { return r.releaseLostCompute(ctx, p, owner, state, err) } @@ -137,9 +144,8 @@ func (r *runtimeLifecycle) idleCompute(ctx context.Context, p sandbox.SandboxPro if compute.Status != "running" || !compute.BootstrapComplete { return sandbox.ErrComputeUnconfirmed } - peer, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID) - if err != nil { - return err + if peerErr != nil { + return peerErr } if _, err := r.deployment.CheckRunning(ctx, owner); err != nil { return err diff --git a/services/core/internal/execution/runtime_compute_wake.go b/services/core/internal/execution/runtime_compute_wake.go index a3578a837..125d5ede8 100644 --- a/services/core/internal/execution/runtime_compute_wake.go +++ b/services/core/internal/execution/runtime_compute_wake.go @@ -6,6 +6,7 @@ import ( "time" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment/placement" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/runtimegateway" @@ -31,7 +32,7 @@ func (r *runtimeLifecycle) wakeCompute(ctx context.Context, p sandbox.SandboxPro // This idempotent control signal is fenced by guest PID/start time and the // suspension token. It cannot execute or replay an agent request. started := time.Now() - result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", "/run/oac/daemon-suspend.json", "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) + result, err := p.RunCommandCompute(ctx, runtimeReference(owner), state.Current, sandbox.Command{Args: []string{"oac-daemon", "resume", "--control-file", runtimebootstrap.SuspendControlFile, "--environment-id", owner.EnvironmentID, "--suspend-id", state.SuspendID}}) logComputeStep(ctx, owner, "wake_command", started, err) if err != nil { return err diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index ff8de1e0e..7a05d2ece 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -109,12 +109,9 @@ func validatedRuntimeProvider(config *RuntimeProvider, registry *runtimegateway. if copied.ProviderKind == "" || (copied.Mode != string(sandbox.DeploymentNodes) && copied.Mode != string(sandbox.DeploymentDirect)) { return RuntimeProvider{}, sandbox.ErrInvalid } - if copied.Mode == string(sandbox.DeploymentDirect) && copied.Suspension != nil { - return RuntimeProvider{}, sandbox.ErrInvalid - } if config.Suspension != nil { policy := *config.Suspension - if !sandbox.SupportsCheckpoint(config.Provider) || policy.Retention < time.Second { + if !sandbox.SupportsCheckpoint(config.Provider.ProviderOperations()) || policy.Retention < time.Second { return RuntimeProvider{}, sandbox.ErrInvalid } copied.Suspension = &policy diff --git a/services/core/internal/execution/runtime_lost_compute_test.go b/services/core/internal/execution/runtime_lost_compute_test.go index 365c47e27..d30cf62e1 100644 --- a/services/core/internal/execution/runtime_lost_compute_test.go +++ b/services/core/internal/execution/runtime_lost_compute_test.go @@ -33,6 +33,9 @@ type lostComputeProvider struct { func (p *lostComputeProvider) GetCompute(_ context.Context, _ sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { return p.observe(c) } +func (p *lostComputeProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, r, c) +} func (p *lostComputeProvider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { p.killed = append(p.killed, c) if p.onKill != nil { diff --git a/services/core/internal/sandbox/docker/operations.go b/services/core/internal/sandbox/docker/operations.go index 0e2ac9c0d..784193167 100644 --- a/services/core/internal/sandbox/docker/operations.go +++ b/services/core/internal/sandbox/docker/operations.go @@ -23,6 +23,7 @@ func Operations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "docker_does_not_support_checkpoints"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "docker_does_not_support_preparation_cache"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "docker_does_not_support_preparation_cache"}, "Observe": {State: providercontract.Supported}, @@ -56,6 +57,9 @@ func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} } +func (p *Provider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: Operations()["RenewCompute"].Reason} +} func (p *Provider) RestorePreparation(context.Context, sandbox.Reference, string) (bool, error) { return false, &providercontract.UnsupportedError{Operation: "RestorePreparation", Reason: Operations()["RestorePreparation"].Reason} } diff --git a/services/core/internal/sandbox/e2b/checkpoint.go b/services/core/internal/sandbox/e2b/checkpoint.go new file mode 100644 index 000000000..4c5242cfc --- /dev/null +++ b/services/core/internal/sandbox/e2b/checkpoint.go @@ -0,0 +1,180 @@ +package e2b + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" +) + +// One E2B sandbox spans every generation of an allocation: pause keeps the +// sandbox and resume reconnects to it. The helper's private receipt records the +// current generation, so only that generation addresses the sandbox. + +func digest(parts ...string) string { + sum := sha256.Sum256([]byte(strings.Join(parts, "\n"))) + return hex.EncodeToString(sum[:]) +} + +// compatibility names the endpoint and installation that own a paused sandbox +// and the template build it runs; a snapshot restores only under the same two. +func (c Config) compatibility() sandbox.CheckpointCompatibility { + apiURL, domain, _ := NormalizeEndpoint(c.APIURL, c.Domain) + return sandbox.CheckpointCompatibility{ArtifactDomain: digest("e2b-endpoint", c.InstallationID, apiURL, domain), ExecutionClass: digest("e2b-template", c.Template)} +} + +func (p *Provider) name(r sandbox.Reference, generation uint64) string { + sum := sha256.Sum256([]byte(p.config.InstallationID + ":" + r.TenantID + ":" + r.EnvironmentID + ":" + r.AllocationID)) + return fmt.Sprintf("oac-%x-g%d", sum[:16], generation) +} + +func (p *Provider) validSnapshot(r sandbox.Reference, s sandbox.SnapshotIdentity) bool { + return s.Compatibility.Validate() == nil && validID(s.OperationID) && s.SourceName == p.name(r, s.SourceGeneration) && + s.Reference == s.SourceName+":s-"+s.OperationID && s.SourceID != "" && s.ID != "" +} + +func (p *Provider) validCompute(r sandbox.Reference, c sandbox.Compute) bool { + if !validReference(r) || c.Name != p.name(r, c.Generation) { + return false + } + if c.Generation == 0 { + return c.RestoredFrom == nil + } + return c.RestoredFrom != nil && p.validSnapshot(r, *c.RestoredFrom) && c.RestoredFrom.SourceGeneration < c.Generation +} + +func (p *Provider) Initial(ctx context.Context, r sandbox.Reference) (sandbox.Compute, error) { + if err := ctx.Err(); err != nil { + return sandbox.Compute{}, err + } + if !validReference(r) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return sandbox.Compute{Name: p.name(r, 0)}, nil +} + +func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *sandbox.SnapshotIdentity) (sandbox.Compute, error) { + if err := ctx.Err(); err != nil { + return sandbox.Compute{}, err + } + c := sandbox.Compute{Generation: generation, Name: p.name(r, generation)} + if snapshot != nil { + value := *snapshot + c.RestoredFrom = &value + } + if !p.validCompute(r, c) { + return sandbox.Compute{}, sandbox.ErrInvalid + } + return c, nil +} + +// state sends one checkpoint request and validates the returned incarnation +// against the one the request named. +func (p *Provider) state(ctx context.Context, q Request, want sandbox.Compute) (sandbox.ComputeState, error) { + deadline, ok := ctx.Deadline() + if !ok || !p.validCompute(q.Reference, want) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + if err := ctx.Err(); err != nil { + return sandbox.ComputeState{}, err + } + q.Version, q.Config, q.Deadline = ProtocolVersion, p.config, deadline + out, err := p.exchange(ctx, q) + if err != nil { + return sandbox.ComputeState{}, err + } + s := out.State + if s == nil { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + if s.RestoreAttemptClosed != "" { + if q.Resume == nil || !s.ClosesRestoreAttempt(*q.Resume) { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + return *s, nil + } + got := s.Compute + if got.ID == "" || got.Name != want.Name || got.Generation != want.Generation || (want.ID != "" && got.ID != want.ID) || + (got.RestoredFrom == nil) != (want.RestoredFrom == nil) || want.RestoredFrom != nil && *got.RestoredFrom != *want.RestoredFrom { + return sandbox.ComputeState{}, sandbox.ErrOwnership + } + if q.Suspend != nil { + snapshot := s.Snapshot + if snapshot == nil { + if !q.Suspend.ObserveOnly || s.SourceStopped || !s.BootstrapComplete || s.Status != "running" && s.Status != "paused" { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + } else if !p.validSnapshot(q.Reference, *snapshot) || snapshot.Compatibility != p.config.Compatibility || snapshot.OperationID != q.Suspend.OperationID || + snapshot.SourceID != want.ID && want.ID != "" || snapshot.SourceID != got.ID || !q.Suspend.ObserveOnly && (!s.SourceStopped || s.Status != "paused") { + return sandbox.ComputeState{}, sandbox.ErrComputeUnconfirmed + } + } + return *s, nil +} + +func (p *Provider) GetCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, Request{Operation: "compute", Reference: r, Compute: &c}, c) +} + +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, Request{Operation: "renew_compute", Reference: r, Compute: &c}, c) +} + +func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: &c}, c) +} + +func (p *Provider) Suspend(ctx context.Context, q sandbox.SuspendRequest) (sandbox.ComputeState, error) { + if !validID(q.OperationID) || q.Snapshot != nil && !p.validSnapshot(q.Reference, *q.Snapshot) { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + return p.state(ctx, Request{Operation: "suspend", Reference: q.Reference, Suspend: &q}, q.Source) +} + +// Resume reconnects to the paused sandbox. A snapshot from another endpoint or +// template build is unusable before any native call. +func (p *Provider) Resume(ctx context.Context, q sandbox.ResumeRequest) (sandbox.ComputeState, error) { + if q.Workspace != nil { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + if !validID(q.OperationID) || !p.validSnapshot(q.Reference, q.Snapshot) || q.Target.RestoredFrom == nil || *q.Target.RestoredFrom != q.Snapshot { + return sandbox.ComputeState{}, sandbox.ErrInvalid + } + if q.Snapshot.Compatibility != p.config.Compatibility { + return sandbox.ComputeState{}, sandbox.ErrCheckpointUnusable + } + return p.state(ctx, Request{Operation: "resume", Reference: q.Reference, Resume: &q}, q.Target) +} + +func (p *Provider) KillCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) error { + deadline, ok := ctx.Deadline() + if !ok || !p.validCompute(r, c) { + return sandbox.ErrInvalid + } + _, err := p.exchange(ctx, Request{Version: ProtocolVersion, Operation: "kill_compute", Config: p.config, Reference: r, Compute: &c, Deadline: deadline}) + return err +} + +func (p *Provider) DeleteSnapshot(ctx context.Context, r sandbox.Reference, s sandbox.SnapshotIdentity) error { + deadline, ok := ctx.Deadline() + if !ok || !validReference(r) || !p.validSnapshot(r, s) { + return sandbox.ErrInvalid + } + _, err := p.exchange(ctx, Request{Version: ProtocolVersion, Operation: "delete_snapshot", Config: p.config, Reference: r, Snapshot: &s, Deadline: deadline}) + return err +} + +func (p *Provider) RunCommandCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute, command sandbox.Command) (sandbox.CommandResult, error) { + if err := validCommand(command); err != nil { + return sandbox.CommandResult{}, err + } + deadline, ok := ctx.Deadline() + if !ok || !p.validCompute(r, c) { + return sandbox.CommandResult{}, sandbox.ErrInvalid + } + out, err := p.exchange(ctx, Request{Version: ProtocolVersion, Operation: "command_compute", Config: p.config, Reference: r, Compute: &c, Command: &command, Deadline: deadline}) + return commandResult(out, err) +} diff --git a/services/core/internal/sandbox/e2b/helper_contract.go b/services/core/internal/sandbox/e2b/helper_contract.go index f4cf30ffb..7beb4c8c0 100644 --- a/services/core/internal/sandbox/e2b/helper_contract.go +++ b/services/core/internal/sandbox/e2b/helper_contract.go @@ -20,12 +20,12 @@ const MaxCommandInputBytes = sandbox.MaxCommandInputBytes // HelperOperations declares the complete set of one-shot helper operations. func HelperOperations() []string { - return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential"} + return []string{"create", "inspect", "renew", "kill", "command", "validate_deployment", "observe", "list_templates", "list_builds", "verify_credential", "compute", "renew_compute", "suspend", "resume", "kill_compute", "delete_snapshot", "command_compute", "resume_compute"} } // HelperErrors are sanitized wire outcomes; an empty code denotes success. func HelperErrors() []string { - return []string{"", "invalid", "ownership", "exists", "not_found", "command_unconfirmed", "unconfirmed", "template_invalid", "team_mismatch", "unauthorized"} + return []string{"", "invalid", "ownership", "exists", "not_found", "command_unconfirmed", "unconfirmed", "template_invalid", "team_mismatch", "unauthorized", "checkpoint_unusable"} } // Validate checks the operation envelope before the helper can start. Native @@ -55,6 +55,11 @@ func (q Request) Validate() error { if !validReference(q.Reference) { return sandbox.ErrInvalid } + present := map[string]bool{"compute": q.Compute != nil, "renew_compute": q.Compute != nil, "kill_compute": q.Compute != nil, "resume_compute": q.Compute != nil, + "command_compute": q.Compute != nil && q.Command != nil, "suspend": q.Suspend != nil, "resume": q.Resume != nil, "delete_snapshot": q.Snapshot != nil} + if want, checkpoint := present[q.Operation]; checkpoint && !want { + return sandbox.ErrInvalid + } } return nil } @@ -69,12 +74,17 @@ type Request struct { Bootstrap *sandbox.Bootstrap `json:",omitempty"` RuntimeBootstrap *runtimebootstrap.Connection `json:",omitempty"` Command *sandbox.Command `json:",omitempty"` + Compute *sandbox.Compute `json:",omitempty"` + Suspend *sandbox.SuspendRequest `json:",omitempty"` + Resume *sandbox.ResumeRequest `json:",omitempty"` + Snapshot *sandbox.SnapshotIdentity `json:",omitempty"` Deadline time.Time } type Response struct { Version int Info *sandbox.Info `json:",omitempty"` Command *sandbox.CommandResult `json:",omitempty"` + State *sandbox.ComputeState `json:",omitempty"` ErrorCode string DeploymentValid bool `json:",omitempty"` TemplateBuild *TemplateBuild `json:",omitempty"` diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 79d48cdd2..90ef24e80 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -81,7 +81,7 @@ func main() { identity = append(identity, "InstallationID") values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, - "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, + "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), @@ -131,6 +131,21 @@ func fixtures() []byte { if operation == "verify_credential" { copy.References = []sandbox.Reference{r} } + compute := sandbox.Compute{Name: "fixture", ID: "fixture-id"} + snapshot := sandbox.SnapshotIdentity{Compatibility: sandbox.CheckpointCompatibility{ArtifactDomain: "a", ExecutionClass: "e"}, Reference: "fixture:s-op", ID: "fixture-id", OperationID: "op", SourceName: "fixture", SourceID: "fixture-id"} + switch operation { + case "compute", "renew_compute", "resume_compute", "kill_compute": + copy.Compute = &compute + case "command_compute": + copy.Compute, copy.Command = &compute, &sandbox.Command{Args: []string{"true"}} + case "suspend": + copy.Suspend = &sandbox.SuspendRequest{Reference: r, OperationID: "op", Source: compute} + case "resume": + target := sandbox.Compute{Generation: 1, Name: "fixture-g1", RestoredFrom: &snapshot} + copy.Resume = &sandbox.ResumeRequest{Reference: r, OperationID: "op", Snapshot: snapshot, Target: target} + case "delete_snapshot": + copy.Snapshot = &snapshot + } add("request", operation, true, copy) } for _, item := range []struct { diff --git a/services/core/internal/sandbox/e2b/operations.go b/services/core/internal/sandbox/e2b/operations.go index 1a6feee22..0fd31f1df 100644 --- a/services/core/internal/sandbox/e2b/operations.go +++ b/services/core/internal/sandbox/e2b/operations.go @@ -14,48 +14,22 @@ func Operations() providercontract.Operations { "Renew": {State: providercontract.Supported}, "Kill": {State: providercontract.Supported}, "RunCommand": {State: providercontract.Supported}, - "Initial": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "NewCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "GetCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Suspend": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "Resume": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "KillCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "RunCommandCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, - "ResumeCompute": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_checkpoints"}, + "Initial": {State: providercontract.Supported}, + "NewCompute": {State: providercontract.Supported}, + "GetCompute": {State: providercontract.Supported}, + "Suspend": {State: providercontract.Supported}, + "Resume": {State: providercontract.Supported}, + "KillCompute": {State: providercontract.Supported}, + "DeleteSnapshot": {State: providercontract.Supported}, + "RunCommandCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, + "ResumeCompute": {State: providercontract.Supported}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_preparation_cache"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "e2b_does_not_support_preparation_cache"}, "Observe": {State: providercontract.Supported}, } } func (*Provider) ProviderOperations() providercontract.Operations { return Operations() } -func (p *Provider) Initial(context.Context, sandbox.Reference) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "Initial", Reason: Operations()["Initial"].Reason} -} -func (p *Provider) NewCompute(context.Context, sandbox.Reference, uint64, *sandbox.SnapshotIdentity) (sandbox.Compute, error) { - return sandbox.Compute{}, &providercontract.UnsupportedError{Operation: "NewCompute", Reason: Operations()["NewCompute"].Reason} -} -func (p *Provider) GetCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "GetCompute", Reason: Operations()["GetCompute"].Reason} -} -func (p *Provider) Suspend(context.Context, sandbox.SuspendRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Suspend", Reason: Operations()["Suspend"].Reason} -} -func (p *Provider) Resume(context.Context, sandbox.ResumeRequest) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "Resume", Reason: Operations()["Resume"].Reason} -} -func (p *Provider) KillCompute(context.Context, sandbox.Reference, sandbox.Compute) error { - return &providercontract.UnsupportedError{Operation: "KillCompute", Reason: Operations()["KillCompute"].Reason} -} -func (p *Provider) DeleteSnapshot(context.Context, sandbox.Reference, sandbox.SnapshotIdentity) error { - return &providercontract.UnsupportedError{Operation: "DeleteSnapshot", Reason: Operations()["DeleteSnapshot"].Reason} -} -func (p *Provider) RunCommandCompute(context.Context, sandbox.Reference, sandbox.Compute, sandbox.Command) (sandbox.CommandResult, error) { - return sandbox.CommandResult{}, &providercontract.UnsupportedError{Operation: "RunCommandCompute", Reason: Operations()["RunCommandCompute"].Reason} -} -func (p *Provider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { - return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: Operations()["ResumeCompute"].Reason} -} func (p *Provider) RestorePreparation(context.Context, sandbox.Reference, string) (bool, error) { return false, &providercontract.UnsupportedError{Operation: "RestorePreparation", Reason: Operations()["RestorePreparation"].Reason} } diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index e420a406c..92ecc5b4b 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -22,6 +22,8 @@ type Config struct { Binary, StateDir, InstallationID, APIKey, Template, APIURL, Domain string TimeoutSeconds int Resources *sandbox.Resources + // Compatibility is derived from the other fields by NewWithCaller. + Compatibility sandbox.CheckpointCompatibility } // Discover uses the same pinned SDK helper without requiring a saved deployment. @@ -149,6 +151,7 @@ func NewWithCaller(c Config, caller Caller) (*Provider, error) { resources := *c.Resources c.Resources = &resources } + c.Compatibility = c.compatibility() return &Provider{config: c, caller: caller, now: time.Now}, nil } @@ -194,12 +197,18 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen } connection = &value } - out, err := p.caller.Call(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) + return p.exchange(ctx, Request{Version: ProtocolVersion, Operation: operation, Config: p.config, Reference: r, Bootstrap: b, RuntimeBootstrap: connection, Command: command, Deadline: deadline}) +} + +// exchange sends one validated helper request and maps its sanitized outcome. +func (p *Provider) exchange(ctx context.Context, q Request) (Response, error) { + operation, r := q.Operation, q.Reference + out, err := p.caller.Call(ctx, q) if errors.Is(err, errHelperNotStarted) { return unstarted(operation, r), sandbox.ErrComputeUnconfirmed } if err != nil || out.Version != ProtocolVersion { - if operation == "command" { + if operation == "command" || operation == "command_compute" { return Response{}, sandbox.ErrCommandUnconfirmed } return Response{}, sandbox.ErrComputeUnconfirmed @@ -224,6 +233,8 @@ func (p *Provider) call(ctx context.Context, operation string, r sandbox.Referen return out, sandbox.ErrExists case "not_found": return out, sandbox.ErrNotFound + case "checkpoint_unusable": + return out, sandbox.ErrCheckpointUnusable case "command_unconfirmed": return out, sandbox.ErrCommandUnconfirmed default: @@ -286,16 +297,18 @@ func (p *Provider) Kill(ctx context.Context, r sandbox.Reference) error { } return err } -func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { +func validCommand(c sandbox.Command) error { if len(c.Args) == 0 || len(c.Stdin) > sandbox.MaxCommandInputBytes || c.Directory != "" && !filepath.IsAbs(c.Directory) { - return sandbox.CommandResult{}, sandbox.ErrInvalid + return sandbox.ErrInvalid } for _, arg := range c.Args { if strings.ContainsRune(arg, 0) { - return sandbox.CommandResult{}, sandbox.ErrInvalid + return sandbox.ErrInvalid } } - out, err := p.call(ctx, "command", r, nil, &c) + return nil +} +func commandResult(out Response, err error) (sandbox.CommandResult, error) { if err != nil { return sandbox.CommandResult{}, err } @@ -304,6 +317,12 @@ func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbo } return *out.Command, nil } +func (p *Provider) RunCommand(ctx context.Context, r sandbox.Reference, c sandbox.Command) (sandbox.CommandResult, error) { + if err := validCommand(c); err != nil { + return sandbox.CommandResult{}, err + } + return commandResult(p.call(ctx, "command", r, nil, &c)) +} // A fresh allocation Create rejected before process startup has no cloud effects. // The common Provider contract forbids replaying an earlier unknown Create. diff --git a/services/core/internal/sandbox/microsandbox/operations.go b/services/core/internal/sandbox/microsandbox/operations.go index 14f788411..acb175140 100644 --- a/services/core/internal/sandbox/microsandbox/operations.go +++ b/services/core/internal/sandbox/microsandbox/operations.go @@ -19,6 +19,7 @@ func Operations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "RestorePreparation": {State: providercontract.Supported}, "SavePreparation": {State: providercontract.Supported}, "Observe": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/microsandbox/provider.go b/services/core/internal/sandbox/microsandbox/provider.go index 37637de66..0053b5295 100644 --- a/services/core/internal/sandbox/microsandbox/provider.go +++ b/services/core/internal/sandbox/microsandbox/provider.go @@ -261,6 +261,11 @@ func (p *Provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c Com return p.state(ctx, Request{Operation: "resume_compute", Reference: r, Compute: c}) } +// RenewCompute observes: a microsandbox VM has no native lease. +func (p *Provider) RenewCompute(ctx context.Context, r sandbox.Reference, c Compute) (State, error) { + return p.GetCompute(ctx, r, c) +} + func (p *Provider) NewCompute(ctx context.Context, r sandbox.Reference, generation uint64, snapshot *SnapshotIdentity) (Compute, error) { if err := ctx.Err(); err != nil { return Compute{}, err diff --git a/services/core/internal/sandbox/node/agent.go b/services/core/internal/sandbox/node/agent.go index 6508929cd..4b3118e51 100644 --- a/services/core/internal/sandbox/node/agent.go +++ b/services/core/internal/sandbox/node/agent.go @@ -402,7 +402,7 @@ func (a *agent) executeWork(ctx context.Context, task work) { func logOperation(ctx context.Context, task work, started, finished time.Time, code string, err error) { level := slog.LevelInfo switch task.request.Operation { - case "info", "renew", "initial", "new_compute", "compute", "observe": + case "info", "renew", "initial", "new_compute", "compute", "renew_compute", "observe": if code == "" { level = slog.LevelDebug } diff --git a/services/core/internal/sandbox/node/generations.go b/services/core/internal/sandbox/node/generations.go index 89e3add8d..132720dad 100644 --- a/services/core/internal/sandbox/node/generations.go +++ b/services/core/internal/sandbox/node/generations.go @@ -354,7 +354,7 @@ func (m *GenerationManager) probeLoop() { } ctx, cancel := context.WithTimeout(m.ctx, 5*time.Second) checkpoint, err := g.value.Probe(ctx) - if err == nil && ((sandbox.SupportsCheckpoint(g.value.Provider) && (checkpoint == nil || checkpoint.Validate() != nil)) || (!sandbox.SupportsCheckpoint(g.value.Provider) && checkpoint != nil)) { + if err == nil && ((sandbox.SupportsCheckpoint(g.value.Provider.ProviderOperations()) && (checkpoint == nil || checkpoint.Validate() != nil)) || (!sandbox.SupportsCheckpoint(g.value.Provider.ProviderOperations()) && checkpoint != nil)) { err = sandbox.ErrInvalid } cancel() diff --git a/services/core/internal/sandbox/node/node_test.go b/services/core/internal/sandbox/node/node_test.go index 113447028..5c360e01d 100644 --- a/services/core/internal/sandbox/node/node_test.go +++ b/services/core/internal/sandbox/node/node_test.go @@ -169,7 +169,7 @@ func TestLostCreateResponseDoesNotReplayAndReconnectSerializesCleanup(t *testing func TestOfflineIsUnknownAndDockerDoesNotAdvertiseCheckpoint(t *testing.T) { h := newTestHub(HubOptions{OwnerEpoch: func(context.Context) (uint64, error) { return 1, nil }}) p := h.Proxy(uuid.NewString(), docker.Operations(), 1) - if sandbox.SupportsCheckpoint(p) { + if sandbox.SupportsCheckpoint(p.ProviderOperations()) { t.Fatal("docker advertised checkpoint") } _, err := p.GetInfo(context.Background(), reference()) diff --git a/services/core/internal/sandbox/node/operations.go b/services/core/internal/sandbox/node/operations.go index 55e9ba504..e62b13f56 100644 --- a/services/core/internal/sandbox/node/operations.go +++ b/services/core/internal/sandbox/node/operations.go @@ -16,6 +16,7 @@ var operationMethods = map[string]string{ "delete_snapshot": "DeleteSnapshot", "command_compute": "RunCommandCompute", "resume_compute": "ResumeCompute", + "renew_compute": "RenewCompute", "restore_preparation": "RestorePreparation", "save_preparation": "SavePreparation", "observe": "Observe", diff --git a/services/core/internal/sandbox/node/provider_operations_fixture_test.go b/services/core/internal/sandbox/node/provider_operations_fixture_test.go index 07bfec1e1..10fe1f6d0 100644 --- a/services/core/internal/sandbox/node/provider_operations_fixture_test.go +++ b/services/core/internal/sandbox/node/provider_operations_fixture_test.go @@ -24,6 +24,7 @@ func (*fakeProvider) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -56,6 +57,9 @@ func (*fakeProvider) RunCommandCompute(context.Context, sandbox.Reference, sandb func (*fakeProvider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: "fixture_operation_not_supported"} } +func (*fakeProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} func (*fakeProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } @@ -75,6 +79,7 @@ func (*observationProvider) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Supported}, diff --git a/services/core/internal/sandbox/node/proxy.go b/services/core/internal/sandbox/node/proxy.go index 4a8b5b34c..07d8b5618 100644 --- a/services/core/internal/sandbox/node/proxy.go +++ b/services/core/internal/sandbox/node/proxy.go @@ -166,6 +166,10 @@ func (p *provider) ResumeCompute(ctx context.Context, r sandbox.Reference, c san return p.state(ctx, request{Operation: "resume_compute", Reference: r, Compute: &c}) } +func (p *provider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + return p.state(ctx, request{Operation: "renew_compute", Reference: r, Compute: &c}) +} + // GenerationProvider routes every operation with allocation-owned generation, // distinct from the request's compute generation. declared is the kind's // registered operations. diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index c12b3cda0..e2f2820b1 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -250,7 +250,7 @@ func (q request) validate() error { if count == 0 || count == 1 && q.Snapshot != nil { return nil } - case "compute", "kill_compute", "resume_compute": + case "compute", "kill_compute", "resume_compute", "renew_compute": if count == 1 && q.Compute != nil { return nil } @@ -345,6 +345,9 @@ func execute(ctx context.Context, p sandbox.SandboxProvider, q request) (respons case "resume_compute": state, err = p.ResumeCompute(ctx, q.Reference, *q.Compute) out.State = &state + case "renew_compute": + state, err = p.RenewCompute(ctx, q.Reference, *q.Compute) + out.State = &state case "command_compute": command, err = p.RunCommandCompute(ctx, q.Reference, *q.Compute, *q.Command) out.Command = &command diff --git a/services/core/internal/sandbox/providers/registration.go b/services/core/internal/sandbox/providers/registration.go index 017965442..9b6363d81 100644 --- a/services/core/internal/sandbox/providers/registration.go +++ b/services/core/internal/sandbox/providers/registration.go @@ -50,11 +50,6 @@ func ValidateRegistration(a Adapter) error { if err := sandbox.ValidateOperations(operations); err != nil { return err } - // The common lifecycle suspends only node allocations, so only a nodes - // registration may declare checkpoint support. - if operations["Initial"].State == providercontract.Supported && a.Mode != sandbox.DeploymentNodes { - return invalid("checkpoint support outside nodes mode") - } if a.Policy.DefaultResources != nil && a.Policy.DefaultResources.ValidatePolicy("", a.Policy) != nil { return invalid("default resources") } diff --git a/services/core/internal/sandbox/providers/registration_test.go b/services/core/internal/sandbox/providers/registration_test.go index 363720fcd..4d7c10cf2 100644 --- a/services/core/internal/sandbox/providers/registration_test.go +++ b/services/core/internal/sandbox/providers/registration_test.go @@ -162,13 +162,13 @@ func TestRegistrationRejectsInvalidDefaultResources(t *testing.T) { } } -// Only a nodes registration may declare checkpoint support. -func TestRegistrationCheckpointRequiresNodes(t *testing.T) { +// Checkpoint support follows the declaration in both modes. +func TestRegistrationAdmitsDirectCheckpoint(t *testing.T) { registry := Builtin() a := registry.adapters["microsandbox"] a.Mode, a.BuildLocal, a.NodeArtifacts, a.BuildDirect = "direct", nil, nil, registry.adapters["e2b"].BuildDirect - if err := ValidateRegistration(a); !errors.Is(err, providercontract.ErrContract) || !strings.Contains(err.Error(), "checkpoint") { - t.Fatal("direct checkpoint registration accepted", err) + if err := ValidateRegistration(a); err != nil { + t.Fatal("direct checkpoint registration rejected", err) } } diff --git a/services/core/internal/sandbox/providers/registry.go b/services/core/internal/sandbox/providers/registry.go index f90ce974c..d797896ed 100644 --- a/services/core/internal/sandbox/providers/registry.go +++ b/services/core/internal/sandbox/providers/registry.go @@ -90,7 +90,7 @@ func (r *Registry) SupportsCheckpoint(kind string) (bool, error) { if err != nil { return false, err } - return a.Operations()["Initial"].State == providercontract.Supported, nil + return sandbox.SupportsCheckpoint(a.Operations()), nil } // RetainedLimit keeps nodes without checkpoint support within their active capacity. @@ -99,7 +99,7 @@ func (r *Registry) RetainedLimit(kind string, active, retained int) (int, error) if err != nil { return 0, err } - if a.Mode == sandbox.DeploymentNodes && a.Operations()["Initial"].State != providercontract.Supported { + if a.Mode == sandbox.DeploymentNodes && !sandbox.SupportsCheckpoint(a.Operations()) { return active, nil } return retained, nil diff --git a/services/core/internal/sandbox/providers/registry_test.go b/services/core/internal/sandbox/providers/registry_test.go index 47a0629bb..10f5be9ba 100644 --- a/services/core/internal/sandbox/providers/registry_test.go +++ b/services/core/internal/sandbox/providers/registry_test.go @@ -19,7 +19,7 @@ func TestRegistrationOwnsDeploymentPolicy(t *testing.T) { }{ {"docker", "nodes", "nodes", false}, {"microsandbox", "nodes", "nodes", true}, - {"e2b", "direct", "e2b", false}, + {"e2b", "direct", "e2b", true}, } { t.Run(tc.kind, func(t *testing.T) { d, err := registry.Describe(tc.kind, installation) diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index 2e0a563a4..b960b04d8 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -125,6 +125,10 @@ type SandboxProvider interface { RunCommandCompute(context.Context, Reference, Compute, Command) (CommandResult, error) // ResumeCompute thaws only the same resident instance after an aborted pause. ResumeCompute(context.Context, Reference, Compute) (ComputeState, error) + // RenewCompute extends the native lease of the exact running incarnation + // where the Provider has one, and otherwise observes exactly like + // GetCompute. It never revives stopped compute. + RenewCompute(context.Context, Reference, Compute) (ComputeState, error) // The preparation cache keeps the Environment filesystem left by one finished // preparation. Key is opaque and the adapter owns storage and lifetime. Both @@ -143,7 +147,7 @@ var requiredOperations = []string{"Create", "GetInfo", "Renew", "Kill", "RunComm // checkpointOperations are all supported or all unsupported: partial cleanup or // restore support cannot safely own a compute incarnation. -var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute"} +var checkpointOperations = []string{"Initial", "NewCompute", "GetCompute", "Suspend", "Resume", "KillCompute", "DeleteSnapshot", "RunCommandCompute", "ResumeCompute", "RenewCompute"} // preparationOperations are both supported or both unsupported. var preparationOperations = []string{"RestorePreparation", "SavePreparation"} @@ -268,7 +272,7 @@ func ValidateOperations(operations providercontract.Operations) error { } } for _, name := range checkpointOperations { - if operations[name].State != operations["Initial"].State { + if (operations[name].State == providercontract.Supported) != SupportsCheckpoint(operations) { return fmt.Errorf("%w: incomplete checkpoint lifecycle", providercontract.ErrContract) } } @@ -278,8 +282,10 @@ func ValidateOperations(operations providercontract.Operations) error { return nil } -func SupportsCheckpoint(p SandboxProvider) bool { - return providercontract.Require(p, "Initial") == nil +// SupportsCheckpoint is the one predicate over a declaration: the checkpoint +// operations are all supported or all unsupported, so Initial decides. +func SupportsCheckpoint(operations providercontract.Operations) bool { + return operations["Initial"].State == providercontract.Supported } // ProcessPaths locates installed adapter helpers and their private state. The diff --git a/services/core/tests/integration/provider_operations_fixture_test.go b/services/core/tests/integration/provider_operations_fixture_test.go index c7d052b56..c68ca0dc0 100644 --- a/services/core/tests/integration/provider_operations_fixture_test.go +++ b/services/core/tests/integration/provider_operations_fixture_test.go @@ -24,6 +24,7 @@ func (*lifecycleProvider) ProviderOperations() providercontract.Operations { "DeleteSnapshot": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RunCommandCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "ResumeCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, + "RenewCompute": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, @@ -56,6 +57,9 @@ func (*lifecycleProvider) RunCommandCompute(context.Context, sandbox.Reference, func (*lifecycleProvider) ResumeCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "ResumeCompute", Reason: "fixture_operation_not_supported"} } +func (*lifecycleProvider) RenewCompute(context.Context, sandbox.Reference, sandbox.Compute) (sandbox.ComputeState, error) { + return sandbox.ComputeState{}, &providercontract.UnsupportedError{Operation: "RenewCompute", Reason: "fixture_operation_not_supported"} +} func (*lifecycleProvider) Observe(context.Context, runtimeobs.Target) (runtimeobs.Sample, error) { return runtimeobs.Sample{}, &providercontract.UnsupportedError{Operation: "Observe", Reason: "fixture_operation_not_supported"} } @@ -75,6 +79,7 @@ func (*fakeCheckpointProvider) ProviderOperations() providercontract.Operations "DeleteSnapshot": {State: providercontract.Supported}, "RunCommandCompute": {State: providercontract.Supported}, "ResumeCompute": {State: providercontract.Supported}, + "RenewCompute": {State: providercontract.Supported}, "RestorePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "SavePreparation": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, "Observe": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}, diff --git a/services/core/tests/integration/runtime_compute_lifecycle_test.go b/services/core/tests/integration/runtime_compute_lifecycle_test.go index 9b164bd5b..2c93db85e 100644 --- a/services/core/tests/integration/runtime_compute_lifecycle_test.go +++ b/services/core/tests/integration/runtime_compute_lifecycle_test.go @@ -38,7 +38,7 @@ type fakeCheckpointProvider struct { captures, restores, captureObservations, restoreObservations int computeKills, snapshotDeletes, wakeCommands int promptFrames atomic.Int32 - quiesces, resumes atomic.Int32 + quiesces, resumes, renews atomic.Int32 loseCapture, loseRestore, rejectQuiesce bool beforeQuiesce func() } @@ -162,6 +162,10 @@ func (p *fakeCheckpointProvider) DeleteSnapshot(_ context.Context, _ sandbox.Ref delete(p.snapshots, s.OperationID) return nil } +func (p *fakeCheckpointProvider) RenewCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { + p.renews.Add(1) + return p.GetCompute(ctx, r, c) +} func (p *fakeCheckpointProvider) ResumeCompute(ctx context.Context, r sandbox.Reference, c sandbox.Compute) (sandbox.ComputeState, error) { state, err := p.GetCompute(ctx, r, c) if err != nil { @@ -273,6 +277,12 @@ type computeLifecycleFixture struct { } func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *computeLifecycleFixture { + return newComputeLifecycleFixtureFor(t, "microsandbox", maxActive, maxRetained) +} + +// newComputeLifecycleFixtureFor runs the fixture on a nodes deployment, or on a +// direct one (e2b) where allocations have no node. +func newComputeLifecycleFixtureFor(t *testing.T, provider string, maxActive, maxRetained int) *computeLifecycleFixture { t.Helper() s, _ := newManagedTestStore(t) registry := runtimegateway.NewRegistry() @@ -288,13 +298,15 @@ func newComputeLifecycleFixture(t *testing.T, maxActive, maxRetained int) *compu } server.Close() }) - f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, "microsandbox"), policy: execution.RuntimeSuspensionPolicy{Retention: time.Hour}} + f := &computeLifecycleFixture{t: t, store: s, provider: p, key: webDeployment(t, s, provider), policy: execution.RuntimeSuspensionPolicy{Retention: time.Hour}} view, err := deploymentService(t, s).View(t.Context()) if err != nil { t.Fatal(err) } f.sql(`UPDATE runtime_deployment SET idle_seconds=30`) - f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + if provider == "microsandbox" { + f.node = enrollNode(t, s, view, deployment.Capacity{MaxActive: maxActive, MaxRetained: maxRetained}).NodeID + } f.start() return f } @@ -303,7 +315,9 @@ func (f *computeLifecycleFixture) start() { t.Helper() w := startWebWorker(t, f.store, f.provider.registry, f.key, f.provider, &f.policy) // The Worker's claim starts a new owner epoch, in which the node reconnects. - onlineManagerNode(t, f.store, f.node) + if f.node != "" { + onlineManagerNode(t, f.store, f.node) + } var once sync.Once stop := func() { once.Do(func() { ctx, cancel := context.WithCancel(context.Background()); cancel(); _ = w.Run(ctx) }) @@ -392,6 +406,28 @@ func TestRuntimeComputeLifecycleIdleSuspendAndQueuedSameSessionWake(t *testing.T } } +// A direct allocation has no node and follows the same suspend and wake flow. +func TestRuntimeComputeLifecycleDirectIdleSuspendAndWake(t *testing.T) { + f := newComputeLifecycleFixtureFor(t, "e2b", 1, 2) + tenant, _, env, owner := f.create() + f.complete(owner) + suspended := f.phase(tenant, env.ID, "suspended") + if f.provider.renews.Load() == 0 { + t.Fatal("running compute was observed without renewing its native lease") + } + if f.provider.captures != 1 || f.provider.computeKills != 1 || len(f.provider.snapshots) != 1 || suspended.NodeID != "" { + t.Fatal("direct capture did not release source compute") + } + f.queued(suspended) + awake := f.phase(tenant, env.ID, "running") + if awake.ID != owner.ID || f.provider.creates != 1 || f.provider.restores != 1 { + t.Fatal("direct wake replaced the allocation") + } + if awake.NodeID != "" { + t.Fatal("direct allocation gained a node") + } +} + func TestRuntimeComputeLifecycleLostCaptureAndRestoreObserveWithoutReplay(t *testing.T) { f := newComputeLifecycleFixture(t, 1, 2) tenant, _, env, owner := f.create() diff --git a/services/core/tests/integration/runtime_wake_hint_integration_test.go b/services/core/tests/integration/runtime_wake_hint_integration_test.go index aedeca68c..c468b4c11 100644 --- a/services/core/tests/integration/runtime_wake_hint_integration_test.go +++ b/services/core/tests/integration/runtime_wake_hint_integration_test.go @@ -38,6 +38,11 @@ func (p *wakeHintScanProvider) GetCompute(ctx context.Context, reference sandbox return p.fakeCheckpointProvider.GetCompute(ctx, reference, compute) } +// A running allocation with a connected Runtime is observed by RenewCompute. +func (p *wakeHintScanProvider) RenewCompute(ctx context.Context, reference sandbox.Reference, compute sandbox.Compute) (sandbox.ComputeState, error) { + return p.GetCompute(ctx, reference, compute) +} + type wakeHintIntegrationTarget struct { tenant string session sessions.Session diff --git a/services/core/tests/integration/sandbox_deployment_view_test.go b/services/core/tests/integration/sandbox_deployment_view_test.go index ca1ed70b7..9f8051f31 100644 --- a/services/core/tests/integration/sandbox_deployment_view_test.go +++ b/services/core/tests/integration/sandbox_deployment_view_test.go @@ -38,7 +38,7 @@ func TestSandboxDeploymentViewRecordsTemplateBuildAndSuspension(t *testing.T) { for _, want := range []string{ `"specification":{"resources":{"cpus":2,"memory_mib":2048}}`, `"template_build":{"status":"ready","resources":{"cpus":2,"memory_mib":2048,"root_disk_mib":24063}}`, - `"suspension":null`, + `"suspension":{"idle_seconds":300,"retention_seconds":86400}`, } { if !bytes.Contains(raw, []byte(want)) { t.Fatalf("E2B view lacks %s: %s", want, raw) diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index 07243a1cb..ad01668fc 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -50,10 +50,14 @@ An unknown Create is never repeated. A Create whose connection material was lost ## Inspection and cleanup -Inspection uses SDK metadata and ID reads only. SDK `connect` is never used because it can resume paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. +Inspection uses SDK metadata and ID reads only. SDK `connect` is used only in Resume, because it resumes paused compute. The version-pinned constructor that restores a client from saved connection material is confined to [`sdk.py`](sdk.py) and covered by a no-connect, no-create test. Resource drift fails inspection but still permits ownership-based cleanup. `CreateSettled` proves that the original Create and bootstrap can no longer mutate; it is independent of `BootstrapComplete`. An empty lookup never settles an unknown Create. Kill destroys every matching sandbox, confirms that none remains and only then records a settled tombstone; it returns `State=absent` with `CreateSettled`. A settled rejected Create with no sandbox IDs proves absence without a cloud request, so GetInfo and Kill still succeed when the key is invalid. Ordinary missing compute has no such proof. +## Checkpoint operations + +One sandbox spans every generation of an allocation. Suspend pauses it with the SDK's native memory snapshot, and Resume reconnects to it with `connect`, so the daemon keeps running and wake uses the suspend control file. The receipt records the current generation, the pause operation and the resume operation. A generation other than the current one is `not_found`, and KillCompute and DeleteSnapshot never touch a sandbox that another generation owns. Resume records the attempt as dispatched before `connect` and never dispatches the same operation again. A snapshot that does not match the paused sandbox, its endpoint or its template build is `checkpoint_unusable` before any native call, and that result is durable for the operation. RenewCompute extends the lease of a running sandbox and otherwise observes like GetCompute. + ## Observations `observe` reads the allocation's sandbox ID from its receipt without taking the allocation lock. It then runs one `GET /sandboxes/metrics` request for that sandbox and one listing of running sandboxes with the allocation's labels concurrently, within the caller's deadline; the listing stops once the sandbox has appeared. Only a sandbox that the listing confirms for exactly that allocation is reported, with the listing's start time. A malformed metrics point is unavailable. Observation never connects to, renews or changes a sandbox and writes no receipt. [Runtime observability](../../../../contracts/agents-api/runtime-observability.md) owns the field mapping. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index ee81a1327..bf7f1aa4a 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,6 +1,6 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" -ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized"] +ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] @@ -9,9 +9,10 @@ MAX_OUTPUT = 1048576 MAX_REQUEST = 75497472 MAX_RESPONSE = 16777216 -OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential"] +OPERATIONS = ["create","inspect","renew","kill","command","validate_deployment","observe","list_templates","list_builds","verify_credential","compute","renew_compute","suspend","resume","kill_compute","delete_snapshot","command_compute","resume_compute"] PROTOCOL_VERSION = 1 REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] -REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Deadline"] -RESPONSE_FIELDS = ["Version","Info","Command","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Snapshot","Deadline"] +RESPONSE_FIELDS = ["Version","Info","Command","State","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] SDK_VERSION = "2.51.0" +SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 76557b500..01f4dc2e5 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -18,6 +18,7 @@ PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +COMPUTE_OPERATIONS = ('compute', 'renew_compute', 'resume_compute', 'suspend', 'resume', 'kill_compute', 'delete_snapshot', 'command_compute') def valid_id(value): @@ -312,6 +313,161 @@ def observe(self): return {'Status': 'unavailable'} return observed(found[0], point) + # One sandbox spans every generation. The receipt's generation says which + # incarnation owns it: pause leaves it as the snapshot and resume advances it. + def generation(self): + return (self.receipt.data or {}).get('generation', 0) + + def current(self, compute): + """The sandbox of the exact current incarnation; any other generation is absent.""" + if compute['Generation'] != self.generation(): + raise Failure('not_found') + cloud = self.inspect() + if cloud is None: + raise Failure('not_found') + if compute.get('ID') and compute['ID'] != cloud.sandbox_id: + raise Failure('ownership') + return cloud + + def state(self, compute, cloud, **extra): + status = getattr(cloud.state, 'value', cloud.state) + return dict(Compute=dict(compute, ID=cloud.sandbox_id), Status=status, + BootstrapComplete=bool((self.receipt.data or {}).get('bootstrap_complete')), **extra) + + def lease(self, cloud): + Sandbox.set_timeout(cloud.sandbox_id, self.config['TimeoutSeconds'], **self.options()) + return self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) + + def get_compute(self): + compute = self.q['Compute'] + return self.state(compute, self.current(compute)) + + def renew_compute(self): + compute = self.q['Compute'] + cloud = self.current(compute) + if cloud.state == 'running': + cloud = self.lease(cloud) + return self.state(compute, cloud) + + def resume_compute(self): + """Thaw only a running source; a paused sandbox is a snapshot, not a source.""" + compute = self.q['Compute'] + cloud = self.current(compute) + if cloud.state != 'running': + raise Failure('unconfirmed') + return self.state(compute, self.lease(cloud)) + + def command_compute(self): + cloud = self.current(self.q['Compute']) + if cloud.state != 'running' or not self.receipt.data.get('bootstrap_complete'): + raise Failure('unconfirmed') + return run(self.client(cloud), self.q['Command'], self.remaining) + + def snapshot(self, operation, source, cloud): + return {'Compatibility': self.config['Compatibility'], 'Reference': source['Name'] + ':s-' + operation, + 'ID': cloud.sandbox_id, 'Digest': '', 'CheckpointID': '', 'CheckpointRoot': '', + 'OperationID': operation, 'SourceGeneration': source['Generation'], 'SourceName': source['Name'], + 'SourceID': cloud.sandbox_id} + + def suspend(self): + request = self.q['Suspend'] + source, operation = request['Source'], request['OperationID'] + cloud = self.current(source) + if cloud.state == 'running' and not request['ObserveOnly']: + # Record the attempt before the native call; pause of a paused sandbox is a no-op. + self.receipt.save(pause={'operation_id': operation, 'status': 'pending'}) + Sandbox.pause(cloud.sandbox_id, **self.options()) + cloud = self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) + pause = (self.receipt.data or {}).get('pause') or {} + if cloud.state != 'paused' or pause.get('operation_id') != operation: + if request['ObserveOnly']: + return self.state(source, cloud) + raise Failure('unconfirmed') + self.receipt.save(pause={'operation_id': operation, 'status': 'paused'}) + return self.state(source, cloud, Snapshot=self.snapshot(operation, source, cloud), SourceStopped=True) + + def close_resume(self, operation, kind): + data = self.receipt.data or {} + closed = dict(data.get('closed', {}), **{operation: kind}) + self.receipt.save(closed=dict(list(closed.items())[-32:]), resume=None) + + def closed_state(self, target, operation): + return {'Compute': target, 'Status': 'absent', 'BootstrapComplete': False, 'RestoreAttemptClosed': operation} + + def finish_resume(self, target, operation, cloud, material=None): + values = {'generation': target['Generation'], 'resume': {'operation_id': operation, 'status': 'resumed'}} + if material is not None: + values['connection'] = material + self.receipt.save(**values) + return self.state(target, cloud) + + def resume(self): + request = self.q['Resume'] + operation, snapshot, target = request['OperationID'], request['Snapshot'], request['Target'] + data = self.receipt.data or {} + kind = data.get('closed', {}).get(operation) + if kind == 'unusable': + raise Failure('checkpoint_unusable') + if kind: + return self.closed_state(target, operation) + record = data.get('resume') or {} + if record.get('operation_id') == operation: + # Native restoration was dispatched: observe it, never dispatch again. + cloud = self.inspect() + if cloud is None: + raise Failure('not_found') + if cloud.state != 'running': + raise Failure('unconfirmed') + return self.finish_resume(target, operation, cloud) + if request['ObserveOnly']: + # No dispatch is recorded, so this attempt can still be closed durably. + self.close_resume(operation, 'attempt') + return self.closed_state(target, operation) + pause = data.get('pause') or {} + cloud = self.inspect() + usable = (cloud is not None and cloud.state == 'paused' and pause.get('status') == 'paused' and + pause.get('operation_id') == snapshot['OperationID'] and + snapshot['SourceGeneration'] == self.generation() < target['Generation'] and + snapshot['SourceID'] == cloud.sandbox_id) + if usable: + try: + self.qualified(cloud) + except Failure: + usable = False + if not usable: + self.close_resume(operation, 'unusable') + raise Failure('checkpoint_unusable') + self.receipt.save(resume={'operation_id': operation, 'target': target['Generation'], 'status': 'dispatched'}) + try: + connected = Sandbox.connect(cloud.sandbox_id, timeout=self.config['TimeoutSeconds'], **self.options()) + except SandboxNotFoundException: + self.close_resume(operation, 'unusable') + raise Failure('checkpoint_unusable') from None + cloud = self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) + if cloud.state != 'running': + raise Failure('unconfirmed') + return self.finish_resume(target, operation, cloud, connection_material(connected)) + + def kill_compute(self): + """Kill only the sandbox's owning incarnation; a superseded one owns nothing.""" + compute = self.q['Compute'] + data = self.receipt.data or {} + record = data.get('resume') or {} + if compute['Generation'] == self.generation() or (record.get('status') == 'dispatched' and record.get('target') == compute['Generation']): + self.kill_owned(compute.get('ID')) + + def delete_snapshot(self): + """A snapshot is the paused sandbox itself; one consumed by resume owns nothing.""" + snapshot = self.q['Snapshot'] + if snapshot['SourceGeneration'] == self.generation(): + self.kill_owned(snapshot['SourceID']) + + def kill_owned(self, sandbox_id): + ids = (self.receipt.data or {}).get('ids', []) + if sandbox_id and ids and sandbox_id not in ids: + raise Failure('ownership') + self.kill() + def metadata_for(self, reference): return {PREFIX + field.lower(): value for field, value in dict(reference, InstallationID=self.config['InstallationID']).items()} @@ -391,12 +547,25 @@ def execute(self): raise Failure('unconfirmed') result = run(self.client(cloud), self.q['Command'], self.remaining) return {'Version': PROTOCOL_VERSION, 'Command': result, 'ErrorCode': ''} + if operation in ('compute', 'renew_compute', 'resume_compute', 'suspend', 'resume'): + handler = {'compute': self.get_compute, 'renew_compute': self.renew_compute, 'resume_compute': self.resume_compute, + 'suspend': self.suspend, 'resume': self.resume}[operation] + return {'Version': PROTOCOL_VERSION, 'State': handler(), 'ErrorCode': ''} + if operation == 'command_compute': + return {'Version': PROTOCOL_VERSION, 'Command': self.command_compute(), 'ErrorCode': ''} + if operation in ('kill_compute', 'delete_snapshot'): + {'kill_compute': self.kill_compute, 'delete_snapshot': self.delete_snapshot}[operation]() + return {'Version': PROTOCOL_VERSION, 'ErrorCode': ''} cloud = {'create': self.create, 'inspect': self.inspect, 'renew': self.renew}[operation]() return {'Version': PROTOCOL_VERSION, 'Info': self.info(cloud, absent=cloud is None), 'ErrorCode': ''} except Failure as error: + if self.q['Operation'] in COMPUTE_OPERATIONS: + return {'Version': PROTOCOL_VERSION, 'ErrorCode': error.code} info = self.info() if error.code == 'not_found': info = dict(self.reference, ProviderID='', State='absent', BootstrapComplete=False, CreateSettled=False) return {'Version': PROTOCOL_VERSION, 'Info': info, 'ErrorCode': error.code} except Exception: + if self.q['Operation'] in COMPUTE_OPERATIONS: + return {'Version': PROTOCOL_VERSION, 'ErrorCode': 'unconfirmed'} return {'Version': PROTOCOL_VERSION, 'Info': self.info(), 'ErrorCode': 'unconfirmed'} diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 3d22f01e1..2fb08b5f4 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -17,7 +17,7 @@ from state import Failure, Receipt -class ProviderTest(unittest.TestCase): +class Fixture(unittest.TestCase): def setUp(self): self.temporary = tempfile.TemporaryDirectory() self.addCleanup(self.temporary.cleanup) @@ -60,6 +60,8 @@ def call(self, operation): def record(self): return json.loads(next(Path(self.temporary.name).glob('*.json')).read_text()) + +class ProviderTest(Fixture): def test_create_recover_and_never_replay(self): result = self.call('create') self.assertEqual(result['ErrorCode'], '') @@ -322,5 +324,78 @@ def test_unknown_command_does_not_replay(self): client.commands.run.assert_called_once() +class CheckpointTest(Fixture): + """One sandbox spans generations: pause keeps it and connect resumes it.""" + + def setUp(self): + super().setUp() + self.assertEqual(self.call('create')['ErrorCode'], '') + self.config['Compatibility'] = {'artifact_domain': 'a', 'execution_class': 'e'} + self.op = str(uuid4()) + + def compute(self, generation=0, restored=None): + return {'Generation': generation, 'Name': 'n%d' % generation, 'ID': 'owned-id', 'RestoredFrom': restored} + + def suspend(self, observe=False): + self.request['Suspend'] = {'Reference': self.reference, 'OperationID': self.op, 'Source': self.compute(), + 'Snapshot': None, 'ObserveOnly': observe} + return self.call('suspend') + + def resume(self, snapshot, observe=False, op=None): + target = self.compute(1, snapshot) + self.request['Resume'] = {'Reference': self.reference, 'OperationID': op or self.op, 'Snapshot': snapshot, + 'Target': target, 'ObserveOnly': observe} + return self.call('resume') + + def paused(self): + self.cloud.state = SandboxState.PAUSED + + def test_suspend_pauses_once_and_resume_advances_the_generation(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + result = self.suspend() + self.assertEqual(result['ErrorCode'], '') + snapshot = result['State']['Snapshot'] + self.assertTrue(result['State']['SourceStopped']) + self.assertEqual(result['State']['Status'], 'paused') + self.assertEqual(self.suspend()['State']['Snapshot'], snapshot) + self.api.pause.assert_called_once() + self.api.connect.side_effect = lambda *a, **k: (setattr(self.cloud, 'state', 'running'), self.cloud)[1] + resumed = self.resume(snapshot) + self.assertEqual(resumed['ErrorCode'], '') + self.assertEqual(resumed['State']['Status'], 'running') + self.assertEqual(self.record()['generation'], 1) + self.api.connect.assert_called_once() + self.resume(snapshot) + self.api.connect.assert_called_once() + + def test_superseded_generation_owns_nothing(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + self.api.connect.side_effect = lambda *a, **k: (setattr(self.cloud, 'state', 'running'), self.cloud)[1] + self.resume(snapshot) + self.request['Snapshot'] = snapshot + self.assertEqual(self.call('delete_snapshot')['ErrorCode'], '') + self.request['Compute'] = self.compute() + self.assertEqual(self.call('compute')['ErrorCode'], 'not_found') + self.assertEqual(self.call('kill_compute')['ErrorCode'], '') + self.api.kill.assert_not_called() + + def test_unusable_snapshot_is_durable_and_never_connects(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + other = str(uuid4()) + self.assertEqual(self.resume(dict(snapshot, OperationID=other))['ErrorCode'], 'checkpoint_unusable') + self.assertEqual(self.resume(snapshot)['ErrorCode'], 'checkpoint_unusable') + self.api.connect.assert_not_called() + + def test_renew_compute_extends_only_a_running_lease(self): + self.request['Compute'] = self.compute() + self.assertEqual(self.call('renew_compute')['ErrorCode'], '') + self.api.set_timeout.assert_called_once() + self.paused() + self.assertEqual(self.call('renew_compute')['State']['Status'], 'paused') + self.api.set_timeout.assert_called_once() + + if __name__ == '__main__': unittest.main() diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 204cb3e29..3fd5d5402 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -12,35 +12,44 @@ {"kind":"managed","name":"valid","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, {"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, {"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"command_compute","payload":{"Version":1,"Operation":"command_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute","payload":{"Version":1,"Operation":"compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":true},"valid":false}, -{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":null},"valid":false}, -{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":2},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"delete_snapshot","payload":{"Version":1,"Operation":"delete_snapshot","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill_compute","payload":{"Version":1,"Operation":"kill_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"renew_compute","payload":{"Version":1,"Operation":"renew_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume","payload":{"Version":1,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Target":{"Generation":1,"Name":"fixture-g1","ID":"","RestoredFrom":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"}},"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume_compute","payload":{"Version":1,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"suspend","payload":{"Version":1,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Source":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Snapshot":null,"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":true},"valid":false}, +{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":null},"valid":false}, +{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":"1"},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":2},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, {"kind":"response","name":"error-","payload":{"Version":1,"ErrorCode":""},"valid":true}, +{"kind":"response","name":"error-checkpoint_unusable","payload":{"Version":1,"ErrorCode":"checkpoint_unusable"},"valid":true}, {"kind":"response","name":"error-command_unconfirmed","payload":{"Version":1,"ErrorCode":"command_unconfirmed"},"valid":true}, {"kind":"response","name":"error-exists","payload":{"Version":1,"ErrorCode":"exists"},"valid":true}, {"kind":"response","name":"error-invalid","payload":{"Version":1,"ErrorCode":"invalid"},"valid":true}, diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index 4dd868c8e..7098bdcb5 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -7,6 +7,7 @@ import ( "encoding/json" "time" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox" wire "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/sandbox/microsandbox" sdk "github.com/superradcompany/microsandbox/sdk/go" @@ -63,7 +64,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { sdk.WithLabels(labels), sdk.WithDetached(), sdk.WithQuietLogs(), sdk.WithNetwork(b.network()), sdk.WithEnv(map[string]string{ "HOME": "/home/runtime", "OAC_RUNTIME_HOME": "/home/runtime/.oac", - "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": "/run/oac/daemon-suspend.json", + "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": runtimebootstrap.SuspendControlFile, })) if e != nil { return wire.Response{}, e From 94d76e1ed8a93211db252949d4c1dedc09b03d2d Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Fri, 23 Oct 2026 21:28:11 +0000 Subject: [PATCH 02/12] Carry suspend control in the Runtime bootstrap --- apps/daemon/internal/cli/connect.go | 13 +-- apps/daemon/internal/cli/connect_bootstrap.go | 10 +- .../internal/cli/connect_bootstrap_test.go | 6 +- .../internal/cli/connect_environment.go | 2 +- apps/daemon/internal/cli/suspend_control.go | 17 ++-- .../cli/suspend_control_linux_test.go | 3 +- .../internal/cli/suspend_process_other.go | 6 +- docs/runtime-bootstrap.md | 5 + docs/zh/runtime-bootstrap.md | 7 +- internal/runtimebootstrap/bootstrap.go | 32 +++--- internal/runtimebootstrap/bootstrap_test.go | 26 ++--- .../deploy/e2b/helper_contract_generated.py | 4 +- services/core/deploy/e2b/managed_init.py | 1 - services/core/deploy/e2b/managed_init_test.py | 6 +- .../internal/execution/runtime_lifecycle.go | 1 + .../internal/sandbox/runtime_bootstrap.go | 1 + .../core/internal/sandbox/sandbox_provider.go | 2 + .../e2b-provider/helper_contract_generated.py | 4 +- .../tools/e2b-provider/testdata/contract.json | 99 ++++++++++--------- .../tools/microsandbox-provider/README.md | 6 +- .../tools/microsandbox-provider/bootstrap.go | 6 +- 21 files changed, 142 insertions(+), 115 deletions(-) diff --git a/apps/daemon/internal/cli/connect.go b/apps/daemon/internal/cli/connect.go index be8a74339..9240239fd 100644 --- a/apps/daemon/internal/cli/connect.go +++ b/apps/daemon/internal/cli/connect.go @@ -69,11 +69,12 @@ func runConnect(ctx *runContext, args []string) error { return fmt.Errorf("connect: %w", err) } var bootstrapped *auth.Profile + var suspendEnabled bool if *bootstrapFile != "" { if *remote != "" || *environment != "" || *credentialFile != "" || *workspace != "" || fs.NArg() != 0 { return errors.New("connect: bootstrap input cannot be combined with enrollment options") } - bootstrapped, err = bootstrapProfile(*bootstrapFile) + bootstrapped, suspendEnabled, err = bootstrapProfile(*bootstrapFile) if err != nil { return err } @@ -118,7 +119,7 @@ func runConnect(ctx *runContext, args []string) error { } } - return mainLoop(ctx, *profile, prof, agentCLIs) + return mainLoop(ctx, *profile, prof, agentCLIs, suspendEnabled) } // spawnBackground forks the daemon into the background. Parent @@ -189,11 +190,11 @@ func spawnBackground(ctx context.Context, rc *runContext, profile string, argv [ // background process. SIGINT / SIGTERM cancels the root context, which // unblocks the read pump and any in-flight Send so the daemon exits // without orphaning agent subprocesses. -func mainLoop(rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery) error { - return mainLoopRemote(context.Background(), rc, profile, prof, agentCLIs, "") +func mainLoop(rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, suspend bool) error { + return mainLoopRemote(context.Background(), rc, profile, prof, agentCLIs, "", suspend) } -func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, remote string) error { +func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof auth.Profile, agentCLIs agentCLIDiscovery, remote string, suspend bool) error { // Route through obs/log so daemon log lines pick up the same // trace_id / span_id auto-injection as the server side — when the // daemon adopts an envelope's trace, every log call under that ctx @@ -229,7 +230,7 @@ func mainLoopRemote(parent context.Context, rc *runContext, profile string, prof registry := agent.NewRegistry() registerAgentKinds(registry, agentCLIs) - control, err := newSuspendControl() + control, err := newSuspendControl(suspend) if err != nil { return err } diff --git a/apps/daemon/internal/cli/connect_bootstrap.go b/apps/daemon/internal/cli/connect_bootstrap.go index 718b9f9a2..3be0f82f1 100644 --- a/apps/daemon/internal/cli/connect_bootstrap.go +++ b/apps/daemon/internal/cli/connect_bootstrap.go @@ -10,17 +10,17 @@ import ( // The launch file is the sole credential source for this connection. Reopening // it on process restart never reads or overwrites an auth profile. -func bootstrapProfile(path string) (*auth.Profile, error) { +func bootstrapProfile(path string) (*auth.Profile, bool, error) { raw, err := runtimefs.ReadPrivatePath(path, runtimebootstrap.MaxBytes) if err != nil { - return nil, errors.New("connect: Runtime bootstrap file unavailable") + return nil, false, errors.New("connect: Runtime bootstrap file unavailable") } input, err := runtimebootstrap.Decode(raw) if err != nil { - return nil, err + return nil, false, err } if err = localworkspace.Bind(input.EnvironmentID, input.SessionID, input.Workspace, localworkspace.CapabilityDirectory); err != nil { - return nil, errors.New("connect: Runtime workspace binding unavailable") + return nil, false, errors.New("connect: Runtime workspace binding unavailable") } - return &auth.Profile{ServerURL: input.CoreURL, RuntimeID: input.DeviceID, RunnerCredential: input.Credential}, nil + return &auth.Profile{ServerURL: input.CoreURL, RuntimeID: input.DeviceID, RunnerCredential: input.Credential}, input.SuspendControl, nil } diff --git a/apps/daemon/internal/cli/connect_bootstrap_test.go b/apps/daemon/internal/cli/connect_bootstrap_test.go index 32983a8fc..9f899062b 100644 --- a/apps/daemon/internal/cli/connect_bootstrap_test.go +++ b/apps/daemon/internal/cli/connect_bootstrap_test.go @@ -45,7 +45,7 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { } t.Cleanup(localworkspace.Unbind) for range 2 { - p, err := bootstrapProfile(path) + p, _, err := bootstrapProfile(path) if err != nil || p.ServerURL != input.CoreURL || p.RuntimeID != input.DeviceID || p.RunnerCredential != input.Credential { t.Fatal("failed bootstrap/restart", err) } @@ -60,13 +60,13 @@ func TestBootstrapConnectionDoesNotReadOrOverwritePrivateProfile(t *testing.T) { if err = os.WriteFile(path, []byte("bootstrap-secret"), 0600); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil || strings.Contains(err.Error(), input.Credential) { + if _, _, err = bootstrapProfile(path); err == nil || strings.Contains(err.Error(), input.Credential) { t.Fatal("invalid input fell back or leaked") } if err = os.Remove(path); err != nil { t.Fatal(err) } - if _, err = bootstrapProfile(path); err == nil { + if _, _, err = bootstrapProfile(path); err == nil { t.Fatal("missing input fell back to private auth") } } diff --git a/apps/daemon/internal/cli/connect_environment.go b/apps/daemon/internal/cli/connect_environment.go index d30994a06..08c98d930 100644 --- a/apps/daemon/internal/cli/connect_environment.go +++ b/apps/daemon/internal/cli/connect_environment.go @@ -166,7 +166,7 @@ func runEnvironmentConnect(parent context.Context, rc *runContext, profile strin return err } prof := auth.Profile{ServerURL: base, RuntimeID: bound.DeviceID, RunnerCredential: credential} - return rejected(mainLoopRemote(parent, rc, profile, prof, discovery, remote)) + return rejected(mainLoopRemote(parent, rc, profile, prof, discovery, remote, false)) } var ( diff --git a/apps/daemon/internal/cli/suspend_control.go b/apps/daemon/internal/cli/suspend_control.go index b6d302d46..5cbc8312f 100644 --- a/apps/daemon/internal/cli/suspend_control.go +++ b/apps/daemon/internal/cli/suspend_control.go @@ -14,10 +14,9 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/apps/daemon/internal/localworkspace" "github.com/MiniMax-AI/OpenAgentCore/internal/agentdaemon/proto" + "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" ) -const suspendControlEnv = "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE" - type suspendIdentity struct { PID int `json:"pid"` StartTime string `json:"start_time"` @@ -34,14 +33,16 @@ type suspendControl struct { signal chan os.Signal } -func newSuspendControl() (*suspendControl, error) { - path := os.Getenv(suspendControlEnv) - if path == "" { +// newSuspendControl publishes the control at runtimebootstrap.SuspendControlFile +// when the bootstrap enables it, and fails the connection when it cannot. +func newSuspendControl(enabled bool) (*suspendControl, error) { + if !enabled { return nil, nil } - if !filepath.IsAbs(path) || filepath.Clean(path) != path { - return nil, errors.New("connect: absolute suspend control file required") - } + return openSuspendControl(runtimebootstrap.SuspendControlFile) +} + +func openSuspendControl(path string) (*suspendControl, error) { info, err := os.Lstat(filepath.Dir(path)) if err != nil || !info.IsDir() || info.Mode().Perm()&0077 != 0 { return nil, errors.New("connect: private suspend control directory required") diff --git a/apps/daemon/internal/cli/suspend_control_linux_test.go b/apps/daemon/internal/cli/suspend_control_linux_test.go index eda9dcff7..53d890936 100644 --- a/apps/daemon/internal/cli/suspend_control_linux_test.go +++ b/apps/daemon/internal/cli/suspend_control_linux_test.go @@ -21,13 +21,12 @@ func TestResumeControlRejectsStaleIdentityAndDiscardsPreParkSignals(t *testing.T t.Fatal(err) } path := filepath.Join(dir, "control.json") - t.Setenv(suspendControlEnv, path) environment := uuid.NewString() if err := localworkspace.Bind(environment, uuid.NewString(), t.TempDir(), t.TempDir()); err != nil { t.Fatal(err) } t.Cleanup(localworkspace.Unbind) - control, err := newSuspendControl() + control, err := openSuspendControl(path) if err != nil { t.Fatal(err) } diff --git a/apps/daemon/internal/cli/suspend_process_other.go b/apps/daemon/internal/cli/suspend_process_other.go index 5d57afa2d..3b7a69155 100644 --- a/apps/daemon/internal/cli/suspend_process_other.go +++ b/apps/daemon/internal/cli/suspend_process_other.go @@ -9,8 +9,6 @@ import ( "os" ) -const suspendControlEnv = "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE" - type suspendIdentity struct { EnvironmentID string SuspendID string @@ -21,8 +19,8 @@ type suspendControl struct { lastResumed *proto.EnvironmentSuspendPayload } -func newSuspendControl() (*suspendControl, error) { - if os.Getenv(suspendControlEnv) != "" { +func newSuspendControl(enabled bool) (*suspendControl, error) { + if enabled { return nil, errors.New("hosted suspension requires Linux") } return nil, nil diff --git a/docs/runtime-bootstrap.md b/docs/runtime-bootstrap.md index ac302218b..5fd98fa34 100644 --- a/docs/runtime-bootstrap.md +++ b/docs/runtime-bootstrap.md @@ -53,11 +53,16 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `environment_id` | Canonical nonzero UUID of the Environment the Runtime serves | | `session_id` | Canonical nonzero UUID of the Session whose native history the Runtime keeps | | `workspace` | Clean absolute directory of the workspace, `runtimebootstrap.RuntimeWorkspace` in the packaged Runtime layout. The Runtime serves it as `runtimebootstrap.PublicWorkspace` (`/workspace`), the path applications and Core use | +| `suspend_control` | Required boolean. Core sets it exactly when the Provider declares [checkpoint support](./sandbox-provider.md), and the Provider forwards it unchanged | The decoder rejects unknown, duplicate, missing and case-aliased fields, other versions and documents larger than `runtimebootstrap.MaxBytes`. Errors never include submitted values. A missing or malformed file fails before the daemon connects. The file is the only authentication input for this launch: the daemon refuses to combine it with self-hosted enrollment options, and reads the credential into memory without saving it to a stored profile. Credentials never go in command arguments, environment variables or receipts. The provider keeps the file for process restarts and removes it only during explicit cleanup of the resources it owns. +## Suspend control + +When `suspend_control` is true, the daemon publishes its suspend control at `runtimebootstrap.SuspendControlFile` and parks for suspension. Core's wake command, `oac-daemon resume --control-file`, reads that file. A Provider that declares checkpoint support prepares the file's directory before it starts the daemon: private to the Runtime account (mode 0700, owned by UID 1000). `oac-daemon connect` fails when `suspend_control` is true and it cannot publish the control, and the control is never enabled by any other setting. + ## Timeouts and limits | Value | Applies to | On breach | diff --git a/docs/zh/runtime-bootstrap.md b/docs/zh/runtime-bootstrap.md index dcd137d1b..a5cc5f88e 100644 --- a/docs/zh/runtime-bootstrap.md +++ b/docs/zh/runtime-bootstrap.md @@ -1,7 +1,7 @@ --- title: "Runtime 引导" source: docs/runtime-bootstrap.md -source_hash: 5a6b2fff6d4f54505022e783ebb4a4c20cc8602935842053416870786ee7bc74 +source_hash: 68bb957984c615d36b941efae570b366b6245f08fe4c76561f8aaa92966e1a4b --- Runtime 引导协议连接沙箱提供商与其启动的托管 Runtime。提供商向 Runtime 交付一个引导文件，该文件是此次启动中 Runtime 唯一的认证输入，也是其 Environment 绑定的唯一来源。 @@ -55,11 +55,16 @@ oac-daemon connect --bootstrap-file /home/runtime/runtime-bootstrap.json | `environment_id` | Runtime 所服务 Environment 的规范非零 UUID | | `session_id` | Runtime 保存其原生历史的 Session 的规范非零 UUID | | `workspace` | 工作区的干净绝对目录；打包的 Runtime 布局中为 `runtimebootstrap.RuntimeWorkspace`。Runtime 将其作为 `runtimebootstrap.PublicWorkspace`（`/workspace`）提供，这是应用与 Core 使用的路径 | +| `suspend_control` | 必填布尔值。当且仅当提供商声明[检查点支持](./sandbox-provider.md)时，Core 将其置为 true，提供商原样转交 | 解码器拒绝未知、重复、缺失和大小写别名字段，拒绝其他版本及超过 `runtimebootstrap.MaxBytes`的文档。错误不包含提交的值。文件缺失或格式错误时，daemon 在连接前失败。 该文件是此次启动唯一的认证输入：daemon 拒绝将其与自托管注册选项组合使用，并将凭据读入内存而不保存到存储的 profile。凭据不进入命令参数、环境变量或回执。提供商为进程重启保留该文件，仅在明确清理自己拥有的资源时删除。 +## 挂起控制 {#suspend-control} + +`suspend_control` 为 true 时，daemon 在 `runtimebootstrap.SuspendControlFile` 发布其挂起控制并为挂起而停放。Core 的唤醒命令 `oac-daemon resume --control-file` 读取该文件。声明检查点支持的提供商在启动 daemon 前准备该文件所在目录：仅 Runtime 账户可访问（权限 0700，属主 UID 1000）。`suspend_control` 为 true 而 `oac-daemon connect` 无法发布该控制时，连接失败；没有任何其他设置可以启用该控制。 + ## 超时与限制 {#timeouts-and-limits} | 数值 | 适用对象 | 超出时 | diff --git a/internal/runtimebootstrap/bootstrap.go b/internal/runtimebootstrap/bootstrap.go index 54d069832..ca3aecf14 100644 --- a/internal/runtimebootstrap/bootstrap.go +++ b/internal/runtimebootstrap/bootstrap.go @@ -15,7 +15,7 @@ import ( "github.com/google/uuid" ) -const Version = 2 +const Version = 3 const MaxBytes = 16 * 1024 // PublicWorkspace is the workspace path applications and Core use for a hosted Environment. @@ -26,8 +26,8 @@ const PublicWorkspace = "/workspace" const RuntimeWorkspace = "/environment/workspace" // SuspendControlFile is where the Runtime daemon publishes its suspend control. -// Its directory is private to the Runtime user. Providers enable the control by -// setting OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE to it, and Core's wake command reads it. +// Its directory is private to the Runtime user. The Runtime publishes it exactly when +// the bootstrap sets SuspendControl, and Core's wake command reads it. const SuspendControlFile = "/run/oac/daemon-suspend.json" var ErrInvalid = errors.New("invalid Runtime bootstrap input") @@ -36,14 +36,16 @@ var ErrInvalid = errors.New("invalid Runtime bootstrap input") // CoreURL includes the machine API base path. The provider owns delivery and // file permissions; Runtime owns validation and its authentication representation. // EnvironmentID, SessionID and Workspace bind the Runtime to its Environment. +// SuspendControl is true exactly when the Provider declares checkpoint support. type Connection struct { - Version int `json:"version"` - CoreURL string `json:"core_url"` - DeviceID string `json:"device_id"` - Credential string `json:"credential"` - EnvironmentID string `json:"environment_id"` - SessionID string `json:"session_id"` - Workspace string `json:"workspace"` + Version int `json:"version"` + CoreURL string `json:"core_url"` + DeviceID string `json:"device_id"` + Credential string `json:"credential"` + EnvironmentID string `json:"environment_id"` + SessionID string `json:"session_id"` + Workspace string `json:"workspace"` + SuspendControl bool `json:"suspend_control"` } func canonicalUUID(value string) bool { @@ -111,6 +113,14 @@ func Decode(raw []byte) (Connection, error) { err = d.Decode(&c.SessionID) case "workspace": err = d.Decode(&c.Workspace) + case "suspend_control": + var enabled *bool + if err = d.Decode(&enabled); err == nil && enabled == nil { + err = ErrInvalid + } + if enabled != nil { + c.SuspendControl = *enabled + } default: return Connection{}, ErrInvalid } @@ -121,7 +131,7 @@ func Decode(raw []byte) (Connection, error) { if token, err = d.Token(); err != nil || token != json.Delim('}') { return Connection{}, ErrInvalid } - if len(seen) != 7 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { + if len(seen) != 8 || d.Decode(new(any)) != io.EOF || c.Validate() != nil { return Connection{}, ErrInvalid } return c, nil diff --git a/internal/runtimebootstrap/bootstrap_test.go b/internal/runtimebootstrap/bootstrap_test.go index 524e8973b..86fe22190 100644 --- a/internal/runtimebootstrap/bootstrap_test.go +++ b/internal/runtimebootstrap/bootstrap_test.go @@ -8,7 +8,7 @@ import ( func TestConnectionRoundTrip(t *testing.T) { input := Connection{Version: Version, CoreURL: "https://core.example/api/v1", DeviceID: "da912024-1543-4242-a2c1-5f4f7ebbc6c7", Credential: "test-credential", - EnvironmentID: "0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11", SessionID: "6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e", Workspace: RuntimeWorkspace} + EnvironmentID: "0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11", SessionID: "6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e", Workspace: RuntimeWorkspace, SuspendControl: true} raw, err := input.Marshal() if err != nil { t.Fatal(err) @@ -20,11 +20,14 @@ func TestConnectionRoundTrip(t *testing.T) { } func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { - good := `{"version":2,"core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret","environment_id":"0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11","session_id":"6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e","workspace":"/environment/workspace"}` + good := `{"version":3,"core_url":"https://core.example/api/v1","device_id":"da912024-1543-4242-a2c1-5f4f7ebbc6c7","credential":"test-secret","environment_id":"0d1f4f1c-6c3e-4e0b-9a52-3e3d8b0e7a11","session_id":"6a1b7c0e-2f4d-4c55-8f6e-9d2a1b3c4d5e","workspace":"/environment/workspace","suspend_control":false}` + if _, err := Decode([]byte(good)); err != nil { + t.Fatal("valid input rejected", err) + } for name, raw := range map[string]string{ "null": "null", "array": "[]", "trailing": good + "{}", - "duplicate": strings.Replace(good, `"version":2`, `"version":2,"version":2`, 1), - "unknown": strings.Replace(good, `"version":2`, `"extra":1,"version":2`, 1), + "duplicate": strings.Replace(good, `"version":3`, `"version":3,"version":3`, 1), + "unknown": strings.Replace(good, `"version":3`, `"extra":1,"version":3`, 1), "case": strings.Replace(good, "credential", "Credential", 1), "old auth": `{"server_url":"https://core.example","runtime_id":"old","runner_credential":"test-secret"}`, "oversized": strings.Repeat(" ", MaxBytes) + good, @@ -36,13 +39,14 @@ func TestConnectionRejectsAmbiguousOrForeignInput(t *testing.T) { }) } for field, values := range map[string][]any{ - "version": {nil, 0, 1, "2"}, - "core_url": {"http://user:pass@core.example/api/v1", "https://core.example/api/v1?", "https://core.example/api/v1#", "https://core.example", "https://core.example/api/v1/", "https://core.example/api/v1?q=1", ""}, - "device_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, - "credential": {nil, "", "test\nsecret", "test\x00secret"}, - "environment_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, - "session_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, - "workspace": {"", "workspace", "/", "/environment/../workspace", "/environment/workspace/"}, + "version": {nil, 0, 2, "3"}, + "suspend_control": {nil, 1, "true"}, + "core_url": {"http://user:pass@core.example/api/v1", "https://core.example/api/v1?", "https://core.example/api/v1#", "https://core.example", "https://core.example/api/v1/", "https://core.example/api/v1?q=1", ""}, + "device_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "credential": {nil, "", "test\nsecret", "test\x00secret"}, + "environment_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "session_id": {"", "invalid", "00000000-0000-0000-0000-000000000000"}, + "workspace": {"", "workspace", "/", "/environment/../workspace", "/environment/workspace/"}, } { for _, value := range values { var input map[string]any diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index bf7f1aa4a..952915c39 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,8 +1,8 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] -MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] -MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] MAX_COMMAND_INPUT = 52428832 MAX_CREDENTIAL_REFERENCES = 32 diff --git a/services/core/deploy/e2b/managed_init.py b/services/core/deploy/e2b/managed_init.py index 7b65fa505..a4e6619bf 100644 --- a/services/core/deploy/e2b/managed_init.py +++ b/services/core/deploy/e2b/managed_init.py @@ -49,7 +49,6 @@ def initialize(): control.parent.mkdir(mode=0o700, parents=True, exist_ok=True) os.chown(control.parent, 1000, 1000) control.parent.chmod(0o700) - environment['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'] = str(control) connection = Path(environment['OAC_RUNTIME_HOME']).parent / 'runtime-bootstrap.json' shared.write_private(connection, payload['RuntimeBootstrap'], owner=1000) source.unlink() diff --git a/services/core/deploy/e2b/managed_init_test.py b/services/core/deploy/e2b/managed_init_test.py index 5cc9cce76..fdfc446a9 100644 --- a/services/core/deploy/e2b/managed_init_test.py +++ b/services/core/deploy/e2b/managed_init_test.py @@ -16,9 +16,9 @@ def payload(): value = {key: str(uuid4()) for key in ['InstallationID', 'TenantID', 'EnvironmentID', 'AllocationID', 'SessionID', 'DeviceID']} - return dict(value, RuntimeBootstrap={'version': 2, 'core_url': 'https://core.example/api/v1', + return dict(value, RuntimeBootstrap={'version': 3, 'core_url': 'https://core.example/api/v1', 'device_id': value['DeviceID'], 'credential': 'private-managed-token', - 'environment_id': value['EnvironmentID'], 'session_id': value['SessionID'], 'workspace': '/environment/workspace'}) + 'environment_id': value['EnvironmentID'], 'session_id': value['SessionID'], 'workspace': '/environment/workspace', 'suspend_control': True}) class ManagedStartupTest(unittest.TestCase): @@ -82,7 +82,7 @@ def exercise(self, failed=False): self.assertNotIn(data['RuntimeBootstrap']['credential'], json.dumps(process.call_args.kwargs['env'])) self.assertEqual(json.loads(connection_file.read_text())['environment_id'], data['EnvironmentID']) self.assertEqual(process.call_args.kwargs['user'], 1000) - self.assertEqual(process.call_args.kwargs['env']['OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE'], str(Path(temporary) / 'run' / 'daemon-suspend.json')) + self.assertTrue((Path(temporary) / 'run').is_dir()) if failed: self.assertFalse((root / 'managed-ready.json').exists()) else: diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index 7a05d2ece..ceb23fd40 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -277,6 +277,7 @@ func (r *runtimeLifecycle) provision(ctx context.Context, tenant, environment, p info, err := provider.Create(ctx, sandbox.Bootstrap{ Reference: runtimeReference(owner), SessionID: owner.SessionID, DeviceID: owner.DeviceID, Workspace: workspace, CoreURL: r.config.CoreURL, Credential: token, + SuspendControl: sandbox.SupportsCheckpoint(provider.ProviderOperations()), }) logComputeStep(ctx, owner, "create", started, err) if info.Reference == runtimeReference(owner) && info.CreateSettled && info.State == "absent" { diff --git a/services/core/internal/sandbox/runtime_bootstrap.go b/services/core/internal/sandbox/runtime_bootstrap.go index a47981658..ca8e3e34a 100644 --- a/services/core/internal/sandbox/runtime_bootstrap.go +++ b/services/core/internal/sandbox/runtime_bootstrap.go @@ -9,5 +9,6 @@ func (b Bootstrap) RuntimeConnection() runtimebootstrap.Connection { Version: runtimebootstrap.Version, CoreURL: b.CoreURL, DeviceID: b.DeviceID, Credential: b.Credential, EnvironmentID: b.EnvironmentID, SessionID: b.SessionID, Workspace: runtimebootstrap.RuntimeWorkspace, + SuspendControl: b.SuspendControl, } } diff --git a/services/core/internal/sandbox/sandbox_provider.go b/services/core/internal/sandbox/sandbox_provider.go index b960b04d8..622b8bd69 100644 --- a/services/core/internal/sandbox/sandbox_provider.go +++ b/services/core/internal/sandbox/sandbox_provider.go @@ -62,6 +62,8 @@ type Bootstrap struct { Workspace *workspacefs.Binding `json:",omitempty"` Reference SessionID, DeviceID, CoreURL, Credential string + // SuspendControl is true exactly when the Provider declares checkpoint support. + SuspendControl bool } // Info describes compute only. Running does not establish daemon authentication, diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index bf7f1aa4a..952915c39 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,8 +1,8 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] -MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] -MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] MAX_COMMAND_INPUT = 52428832 MAX_CREDENTIAL_REFERENCES = 32 diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 3fd5d5402..8d07436b1 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -1,53 +1,54 @@ [ -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllocationID","payload":{"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, -{"kind":"managed","name":"valid","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, -{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, -{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"command_compute","payload":{"Version":1,"Operation":"command_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"compute","payload":{"Version":1,"Operation":"compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"delete_snapshot","payload":{"Version":1,"Operation":"delete_snapshot","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"kill_compute","payload":{"Version":1,"Operation":"kill_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"renew_compute","payload":{"Version":1,"Operation":"renew_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"resume","payload":{"Version":1,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Target":{"Generation":1,"Name":"fixture-g1","ID":"","RestoredFrom":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"}},"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"resume_compute","payload":{"Version":1,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"suspend","payload":{"Version":1,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Source":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Snapshot":null,"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, -{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only"},"RuntimeBootstrap":{"version":2,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace"},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, -{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":true},"valid":false}, -{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":null},"valid":false}, -{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":"1"},"valid":false}, -{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":2},"valid":false}, -{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":true}, -{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","version":2,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllocationID","payload":{"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-SuspendControl","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false},"valid":false}, +{"kind":"managed","name":"valid","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, +{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, +{"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"command_compute","payload":{"Version":1,"Operation":"command_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"compute","payload":{"Version":1,"Operation":"compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"config-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":null,"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"create","payload":{"Version":1,"Operation":"create","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"delete_snapshot","payload":{"Version":1,"Operation":"delete_snapshot","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"extra","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Extra":true,"Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"inspect","payload":{"Version":1,"Operation":"inspect","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill","payload":{"Version":1,"Operation":"kill","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"kill_compute","payload":{"Version":1,"Operation":"kill_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_builds","payload":{"Version":1,"Operation":"list_builds","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"list_templates","payload":{"Version":1,"Operation":"list_templates","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"observe","payload":{"Version":1,"Operation":"observe","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"operation-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":null,"Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"operation-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"unsupported","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"reference-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":null,"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"reference-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"renew","payload":{"Version":1,"Operation":"renew","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"renew_compute","payload":{"Version":1,"Operation":"renew_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume","payload":{"Version":1,"Operation":"resume","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Resume":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Snapshot":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"},"Target":{"Generation":1,"Name":"fixture-g1","ID":"","RestoredFrom":{"Compatibility":{"artifact_domain":"a","execution_class":"e"},"Reference":"fixture:s-op","ID":"fixture-id","Digest":"","CheckpointID":"","CheckpointRoot":"","OperationID":"op","SourceGeneration":0,"SourceName":"fixture","SourceID":"fixture-id"}},"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"resume_compute","payload":{"Version":1,"Operation":"resume_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"suspend","payload":{"Version":1,"Operation":"suspend","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Suspend":{"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"OperationID":"op","Source":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Snapshot":null,"ObserveOnly":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"validate_deployment","payload":{"Version":1,"Operation":"validate_deployment","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-0","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-32","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, +{"kind":"request","name":"verify_credential-count-33","payload":{"Version":1,"Operation":"verify_credential","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"References":[{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000001-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000002-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000003-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000004-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000005-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000006-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000007-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000008-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000009-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000000f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000010-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000011-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000012-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000013-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000014-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000015-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000016-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000017-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000018-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000019-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001a-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001b-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001c-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001d-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001e-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"0000001f-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000020-3333-4333-8333-333333333333"},{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"00000021-3333-4333-8333-333333333333"}],"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":{},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":"invalid","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"verify_credential-references-type","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"verify_credential","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"References":[null],"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, +{"kind":"request","name":"version-bool","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":true},"valid":false}, +{"kind":"request","name":"version-null","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":null},"valid":false}, +{"kind":"request","name":"version-string","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":"1"},"valid":false}, +{"kind":"request","name":"version-unknown","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":2},"valid":false}, +{"kind":"request","name":"workspace-\u003cnil\u003e","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":true}, +{"kind":"request","name":"workspace-map[]","payload":{"Bootstrap":{"AllocationID":"33333333-3333-4333-8333-333333333333","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"Config":{"APIKey":"","APIURL":"","Binary":"","Compatibility":{"artifact_domain":"","execution_class":""},"Domain":"","InstallationID":"66666666-6666-4666-8666-666666666666","Resources":null,"StateDir":"","Template":"","TimeoutSeconds":0},"Deadline":"2099-01-01T00:00:00Z","Operation":"create","Reference":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","TenantID":"11111111-1111-4111-8111-111111111111"},"RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"Version":1},"valid":false}, {"kind":"response","name":"error-","payload":{"Version":1,"ErrorCode":""},"valid":true}, {"kind":"response","name":"error-checkpoint_unusable","payload":{"Version":1,"ErrorCode":"checkpoint_unusable"},"valid":true}, {"kind":"response","name":"error-command_unconfirmed","payload":{"Version":1,"ErrorCode":"command_unconfirmed"},"valid":true}, diff --git a/services/core/tools/microsandbox-provider/README.md b/services/core/tools/microsandbox-provider/README.md index e8dec91df..ac6bac559 100644 --- a/services/core/tools/microsandbox-provider/README.md +++ b/services/core/tools/microsandbox-provider/README.md @@ -22,7 +22,7 @@ Create names the VM from a hash of the installation and allocation reference plu Workspace, staging and outputs share the `/environment` filesystem, which keeps the Runtime's cross-device and link checks intact; the layered root filesystem can report different device IDs for a directory and its upper-layer files, so it holds no workspace data. The filesystem provider owns the directory; sandbox removal never deletes it. The binding and resource rules belong to [Workspace filesystem protocol](../../../../docs/workspace-provider.md#attachment). The [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) owns the Runtime directory layout. -VM creation does not run the image's entry point. The bootstrap runs as root through confidential standard input, with a two-minute limit. It creates the Runtime directories and the private control directory `/run/oac` (mode 0700, owned by UID 1000), writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json`, bind-mounts `/environment/workspace` at `/workspace` and starts `oac-daemon connect --bootstrap-file` in the background as UID/GID 1000. `OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE=/run/oac/daemon-suspend.json` enables the daemon's park and wake control. The `io.oac.bootstrap` label then changes from `pending` to `complete` through the SDK's next-start modification policy, because v0.7.8 cannot update the labels of a running VM. That label confirms only these writes and the launch, not authentication or native readiness. +VM creation does not run the image's entry point. The bootstrap runs as root through confidential standard input, with a two-minute limit. It creates the Runtime directories and the private suspend control directory, writes the [Runtime bootstrap](../../../../docs/runtime-bootstrap.md) file to `/home/runtime/runtime-bootstrap.json`, bind-mounts `/environment/workspace` at `/workspace` and starts `oac-daemon connect --bootstrap-file` in the background as UID/GID 1000. The `io.oac.bootstrap` label then changes from `pending` to `complete` through the SDK's next-start modification policy, because v0.7.8 cannot update the labels of a running VM. That label confirms only these writes and the launch, not authentication or native readiness. A helper response carries `CreateSettled` with a configuration rejection only after native Create has completed, the first inspection has verified the exact compute ID and ownership, and the resource check has rejected the VM before bootstrap started. The adapter keeps the original error and validates the compute identity before passing the proof to Core. The private `initial_info` operation used by `GetInfo` and `Renew` can also return an exact initial `State` with `Status="absent"`, no native ID and `CreateSettled=true`, after typed native absence and durable closure of initial Create admission under the allocation lock. It accepts only generation zero without restored ancestry or a native ID. The adapter validates the complete receipt and returns absence without an error. Ordinary `inspect` used by `GetCompute`, uncertain Create outcomes, timeouts and ownership failures never produce this receipt; missing compute alone is not settlement. @@ -53,11 +53,11 @@ The same lock file fences a Create helper that has not yet acquired its lock, in `RunCommand` and `RunCommandCompute` pass standard input on an anonymous pipe, run as UID 1000 with a deadline and a 1 MiB limit per output stream, and return a result only after the input is fully written and the guest reports its exit. Timeouts, output overflow and missing receipts return `ErrCommandUnconfirmed`; closing an SDK exec handle does not prove that the guest process exited ([`command.go`](command.go)). Core uses `RunCommandCompute` only to run the exact wake command it authorizes: ```sh -oac-daemon resume --control-file /run/oac/daemon-suspend.json \ +oac-daemon resume --control-file CONTROL_FILE \ --environment-id ENVIRONMENT_ID --suspend-id SUSPENSION_ID ``` -The command checks the protected Environment and suspension identities and the parked daemon's PID and start time, then signals it through a Linux pidfd. The daemon reauthenticates and waits for Core's resume confirmation before it admits work. +`CONTROL_FILE` is the [suspend control file](../../../../docs/runtime-bootstrap.md#suspend-control). The command checks the protected Environment and suspension identities and the parked daemon's PID and start time, then signals it through a Linux pidfd. The daemon reauthenticates and waits for Core's resume confirmation before it admits work. ## Metrics diff --git a/services/core/tools/microsandbox-provider/bootstrap.go b/services/core/tools/microsandbox-provider/bootstrap.go index 7098bdcb5..21c294124 100644 --- a/services/core/tools/microsandbox-provider/bootstrap.go +++ b/services/core/tools/microsandbox-provider/bootstrap.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "path" "time" "github.com/MiniMax-AI/OpenAgentCore/internal/runtimebootstrap" @@ -18,7 +19,7 @@ import ( const bootstrapScript = ` import ctypes,json,os,stat,subprocess,sys b=json.load(sys.stdin) -for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages','/run/oac']: +for p in ['/home/runtime','/home/runtime/.oac','/environment','/environment/workspace','/environment/staging','/environment/initialization','/environment/packages',sys.argv[1]]: os.makedirs(p,mode=0o700,exist_ok=True) if not stat.S_ISDIR(os.lstat(p).st_mode): raise RuntimeError('invalid bootstrap directory') os.chmod(p,0o700);os.chown(p,1000,1000) @@ -64,7 +65,6 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { sdk.WithLabels(labels), sdk.WithDetached(), sdk.WithQuietLogs(), sdk.WithNetwork(b.network()), sdk.WithEnv(map[string]string{ "HOME": "/home/runtime", "OAC_RUNTIME_HOME": "/home/runtime/.oac", - "OAC_RUNTIME_DAEMON_SUSPEND_PID_FILE": runtimebootstrap.SuspendControlFile, })) if e != nil { return wire.Response{}, e @@ -92,7 +92,7 @@ func (b backend) create(ctx context.Context) (wire.Response, error) { } initialization, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() - result, e := runCommand(initialization, live, sandbox.Command{Args: []string{"/usr/bin/python3", "-I", "-S", "-c", bootstrapScript}, Stdin: data}, "0:0") + result, e := runCommand(initialization, live, sandbox.Command{Args: []string{"/usr/bin/python3", "-I", "-S", "-c", bootstrapScript, path.Dir(runtimebootstrap.SuspendControlFile)}, Stdin: data}, "0:0") if e != nil { return wire.Response{}, e } From 78cde1ec991ab2410d3f60d2c28ec98ae66600f0 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Fri, 23 Oct 2026 21:29:16 +0000 Subject: [PATCH 03/12] Bump the node wire version for renew_compute --- services/core/internal/sandbox/node/wire.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/internal/sandbox/node/wire.go b/services/core/internal/sandbox/node/wire.go index e2f2820b1..b024a229c 100644 --- a/services/core/internal/sandbox/node/wire.go +++ b/services/core/internal/sandbox/node/wire.go @@ -19,7 +19,7 @@ import ( "github.com/gorilla/websocket" ) -const ProtocolVersion = 7 +const ProtocolVersion = 8 const MaxControlFrameBytes = 32 * 1024 const MaxFrameBytes = 72 * 1024 * 1024 const maxPending = 32 From 34b77e3ff394e5f299deb00f227835c7b3d9a833 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Fri, 23 Oct 2026 21:32:42 +0000 Subject: [PATCH 04/12] Fix the E2B checkpoint outcomes and helper docs --- .../deploy/e2b/helper_contract_generated.py | 4 +-- .../sandbox/e2b/internal/contractgen/main.go | 5 ++-- services/core/tools/e2b-provider/README.md | 10 ++++--- .../e2b-provider/helper_contract_generated.py | 4 +-- services/core/tools/e2b-provider/provider.py | 19 +++++++------ .../core/tools/e2b-provider/provider_test.py | 23 ++++++++++++++-- .../tools/e2b-provider/testdata/contract.json | 27 +++++++++---------- 7 files changed, 58 insertions(+), 34 deletions(-) diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index 952915c39..bf7f1aa4a 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -1,8 +1,8 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] -MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] -MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] MAX_COMMAND_INPUT = 52428832 MAX_CREDENTIAL_REFERENCES = 32 diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index 90ef24e80..fd45fc5cc 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -67,9 +67,9 @@ func main() { write("services/core/internal/sandbox/e2b/helper_sdk_generated.go", goCode) bootstrap := fields(reflect.TypeFor[sandbox.Bootstrap]()) requiredBootstrap := fieldNames(reflect.TypeFor[sandbox.Bootstrap](), true) - requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + requiredBootstrap = slices.DeleteFunc(requiredBootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "SuspendControl" }) requiredBootstrap = append(requiredBootstrap, "InstallationID", "RuntimeBootstrap") - bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" }) + bootstrap = slices.DeleteFunc(bootstrap, func(s string) bool { return s == "CoreURL" || s == "Credential" || s == "SuspendControl" }) bootstrap = append(bootstrap, "InstallationID", "RuntimeBootstrap") identity := fields(reflect.TypeFor[sandbox.Reference]()) for _, name := range []string{"SessionID", "DeviceID"} { @@ -197,6 +197,7 @@ func fixtures() []byte { managed := object(b) delete(managed, "CoreURL") delete(managed, "Credential") + delete(managed, "SuspendControl") managed["InstallationID"] = installation managed["RuntimeBootstrap"] = runtimebootstrap.Connection(connection) add("managed", "valid", true, object(managed)) diff --git a/services/core/tools/e2b-provider/README.md b/services/core/tools/e2b-provider/README.md index ad01668fc..6672d41e0 100644 --- a/services/core/tools/e2b-provider/README.md +++ b/services/core/tools/e2b-provider/README.md @@ -17,6 +17,12 @@ The account key is stored encrypted in Core's database and is write-only. It rea | `renew` | `Renew` | Extends the lease of the running sandbox to the configured timeout, then rereads it | | `kill` | `Kill` | Destroys every matching sandbox and confirms that none remains | | `command` | `RunCommand` | Runs one bounded command as the Runtime user on a running sandbox whose bootstrap completed; output is limited to 1 MiB per stream | +| `compute`, `renew_compute` | `GetCompute`, `RenewCompute` | One sandbox spans every generation of an allocation, and the receipt records the current generation, the pause and the resume. Any other generation is `not_found`. `renew_compute` extends a running sandbox's lease and otherwise observes like `compute` | +| `suspend` | `Suspend` | Pauses the sandbox with the SDK's native memory snapshot, so the daemon survives and the paused sandbox is the snapshot. A pause that is dispatched but not seen complete is `unconfirmed`, never running | +| `resume` | `Resume` | Records the attempt as dispatched, then reconnects with `connect`; the same operation is never dispatched twice and a failure after dispatch is `unconfirmed`. A snapshot that does not match the paused sandbox, its endpoint or its template build is `checkpoint_unusable` before any native call, durably for the operation | +| `resume_compute` | `ResumeCompute` | Thaws the source only while it is running | +| `kill_compute`, `delete_snapshot` | `KillCompute`, `DeleteSnapshot` | Destroy the sandbox only for the generation that owns it; a superseded generation owns nothing | +| `command_compute` | `RunCommandCompute` | Runs one bounded command, such as the wake command, on the running current sandbox | | `validate_deployment` | Deployment setup | Reads the template's builds and requires the exact build to be ready with the configured CPU and memory. Without configured resources the selection adopts the build's CPU and memory. Returns the build's status, CPU, memory and reported disk for Core to record; bounded to 30 seconds | | `list_templates`, `list_builds` | [Configuration discovery](../../../../contracts/agents-api/sandbox-deployment.md#configuration-discovery) | Pages the key's visible templates (`GET /v2/templates`) or one template's ready builds, with a transient key. Results are capped at 200 and write no receipt | | `observe` | Runtime observations | One allocation; see [Observations](#observations) | @@ -54,10 +60,6 @@ Inspection uses SDK metadata and ID reads only. SDK `connect` is used only in Re `CreateSettled` proves that the original Create and bootstrap can no longer mutate; it is independent of `BootstrapComplete`. An empty lookup never settles an unknown Create. Kill destroys every matching sandbox, confirms that none remains and only then records a settled tombstone; it returns `State=absent` with `CreateSettled`. A settled rejected Create with no sandbox IDs proves absence without a cloud request, so GetInfo and Kill still succeed when the key is invalid. Ordinary missing compute has no such proof. -## Checkpoint operations - -One sandbox spans every generation of an allocation. Suspend pauses it with the SDK's native memory snapshot, and Resume reconnects to it with `connect`, so the daemon keeps running and wake uses the suspend control file. The receipt records the current generation, the pause operation and the resume operation. A generation other than the current one is `not_found`, and KillCompute and DeleteSnapshot never touch a sandbox that another generation owns. Resume records the attempt as dispatched before `connect` and never dispatches the same operation again. A snapshot that does not match the paused sandbox, its endpoint or its template build is `checkpoint_unusable` before any native call, and that result is durable for the operation. RenewCompute extends the lease of a running sandbox and otherwise observes like GetCompute. - ## Observations `observe` reads the allocation's sandbox ID from its receipt without taking the allocation lock. It then runs one `GET /sandboxes/metrics` request for that sandbox and one listing of running sandboxes with the allocation's labels concurrently, within the caller's deadline; the listing stops once the sandbox has appeared. Only a sandbox that the listing confirms for exactly that allocation is reported, with the listing's start time. A malformed metrics point is unavailable. Observation never connects to, renews or changes a sandbox and writes no receipt. [Runtime observability](../../../../contracts/agents-api/runtime-observability.md) owns the field mapping. diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index 952915c39..bf7f1aa4a 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -1,8 +1,8 @@ # Code generated by contractgen; DO NOT EDIT. """Adapter-private wire declarations. No SDK or repository dependency.""" ERROR_CODES = ["","invalid","ownership","exists","not_found","command_unconfirmed","unconfirmed","template_invalid","team_mismatch","unauthorized","checkpoint_unusable"] -MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] -MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","SuspendControl","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_FIELDS = ["Workspace","TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] +MANAGED_BOOTSTRAP_REQUIRED_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID","RuntimeBootstrap"] MANAGED_IDENTITY_FIELDS = ["TenantID","EnvironmentID","AllocationID","SessionID","DeviceID","InstallationID"] MAX_COMMAND_INPUT = 52428832 MAX_CREDENTIAL_REFERENCES = 32 diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 01f4dc2e5..082e71891 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -1,4 +1,4 @@ -"""Five bounded SDK operations for an already authorized Core allocation, plus +"""Bounded SDK operations for an already authorized Core allocation, plus read-only deployment validation and observation.""" from concurrent.futures import ThreadPoolExecutor import json @@ -176,7 +176,7 @@ def client(self, cloud): raise Failure('unconfirmed') return restore(material, self.options()) - def inspect(self): + def inspect(self, qualify=True): if self.rejected_absence(): return None found = self.discover() @@ -187,7 +187,8 @@ def inspect(self): if len(found) != 1: raise Failure('unconfirmed') cloud = found[0] - self.qualified(cloud) + if qualify: + self.qualified(cloud) record = self.receipt.data or {} if (cloud.state == 'running' and not record.get('bootstrap_complete') and record.get('connection') and record.get('status') not in ('bootstrap_failed', 'killed')): @@ -242,7 +243,7 @@ def create(self): raise Failure('template_invalid' if check['ExitCode'] == 78 else 'unconfirmed') payload = dict(bootstrap, InstallationID=self.config['InstallationID'], RuntimeBootstrap=self.q['RuntimeBootstrap']) - del payload['CoreURL'], payload['Credential'] + del payload['CoreURL'], payload['Credential'], payload['SuspendControl'] if (set(payload) - set(MANAGED_BOOTSTRAP_FIELDS) or not set(MANAGED_BOOTSTRAP_REQUIRED_FIELDS) <= set(payload)): raise Failure('invalid') @@ -380,7 +381,8 @@ def suspend(self): cloud = self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) pause = (self.receipt.data or {}).get('pause') or {} if cloud.state != 'paused' or pause.get('operation_id') != operation: - if request['ObserveOnly']: + # A dispatched pause may still take effect, so running is never reported for it. + if request['ObserveOnly'] and not (pause.get('operation_id') == operation and pause.get('status') == 'pending'): return self.state(source, cloud) raise Failure('unconfirmed') self.receipt.save(pause={'operation_id': operation, 'status': 'paused'}) @@ -424,7 +426,8 @@ def resume(self): self.close_resume(operation, 'attempt') return self.closed_state(target, operation) pause = data.get('pause') or {} - cloud = self.inspect() + # Drift is judged below, before dispatch, as an unusable checkpoint. + cloud = self.inspect(qualify=False) usable = (cloud is not None and cloud.state == 'paused' and pause.get('status') == 'paused' and pause.get('operation_id') == snapshot['OperationID'] and snapshot['SourceGeneration'] == self.generation() < target['Generation'] and @@ -441,8 +444,8 @@ def resume(self): try: connected = Sandbox.connect(cloud.sandbox_id, timeout=self.config['TimeoutSeconds'], **self.options()) except SandboxNotFoundException: - self.close_resume(operation, 'unusable') - raise Failure('checkpoint_unusable') from None + # Dispatched: the outcome is unknown and the attempt stays open. + raise Failure('unconfirmed') from None cloud = self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) if cloud.state != 'running': raise Failure('unconfirmed') diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 2fb08b5f4..5118fdcca 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -27,9 +27,9 @@ def setUp(self): self.request = {'Version': 1, 'Operation': 'create', 'Config': self.config, 'Reference': self.reference, 'Deadline': (datetime.now(timezone.utc) + timedelta(seconds=30)).isoformat(), 'Bootstrap': dict(self.reference, SessionID=str(uuid4()), DeviceID=str(uuid4()), - CoreURL='https://core.example/api/v1', Credential='private-runtime-secret')} + CoreURL='https://core.example/api/v1', Credential='private-runtime-secret', SuspendControl=True)} self.request['RuntimeBootstrap'] = { - 'version': 2, 'core_url': self.request['Bootstrap']['CoreURL'], + 'version': 3, 'suspend_control': True, 'core_url': self.request['Bootstrap']['CoreURL'], 'device_id': self.request['Bootstrap']['DeviceID'], 'credential': self.request['Bootstrap']['Credential']} self.cloud = Mock(sandbox_id='owned-id', sandbox_domain='e2b.app', _envd_version='0.5.0', @@ -388,6 +388,25 @@ def test_unusable_snapshot_is_durable_and_never_connects(self): self.assertEqual(self.resume(snapshot)['ErrorCode'], 'checkpoint_unusable') self.api.connect.assert_not_called() + def test_unseen_pause_never_reports_running(self): + self.assertEqual(self.suspend()['ErrorCode'], 'unconfirmed') + self.assertEqual(self.suspend(observe=True)['ErrorCode'], 'unconfirmed') + + def test_drift_before_dispatch_is_unusable(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + self.cloud.template_id = 'drifted' + self.assertEqual(self.resume(snapshot)['ErrorCode'], 'checkpoint_unusable') + self.api.connect.assert_not_called() + + def test_failure_after_dispatch_stays_open(self): + self.api.pause.side_effect = lambda *a, **k: self.paused() + snapshot = self.suspend()['State']['Snapshot'] + self.api.connect.side_effect = SandboxNotFoundException() + self.assertEqual(self.resume(snapshot)['ErrorCode'], 'unconfirmed') + self.assertEqual(self.record()['resume']['status'], 'dispatched') + self.assertNotIn(self.op, self.record().get('closed', {})) + def test_renew_compute_extends_only_a_running_lease(self): self.request['Compute'] = self.compute() self.assertEqual(self.call('renew_compute')['ErrorCode'], '') diff --git a/services/core/tools/e2b-provider/testdata/contract.json b/services/core/tools/e2b-provider/testdata/contract.json index 8d07436b1..f04757e41 100644 --- a/services/core/tools/e2b-provider/testdata/contract.json +++ b/services/core/tools/e2b-provider/testdata/contract.json @@ -1,18 +1,17 @@ [ -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-AllocationID","payload":{"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-SuspendControl","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, -{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false},"valid":false}, -{"kind":"managed","name":"valid","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, -{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, -{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","SuspendControl":false,"TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":null,"EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"invalid","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"invalid-Extra","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","Extra":true,"InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-AllocationID","payload":{"DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-DeviceID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-EnvironmentID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-InstallationID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-RuntimeBootstrap","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-SessionID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"TenantID":"11111111-1111-4111-8111-111111111111"},"valid":false}, +{"kind":"managed","name":"missing-TenantID","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444"},"valid":false}, +{"kind":"managed","name":"valid","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111"},"valid":true}, +{"kind":"managed","name":"workspace-\u003cnil\u003e","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":null},"valid":true}, +{"kind":"managed","name":"workspace-map[]","payload":{"AllocationID":"33333333-3333-4333-8333-333333333333","DeviceID":"55555555-5555-4555-8555-555555555555","EnvironmentID":"22222222-2222-4222-8222-222222222222","InstallationID":"66666666-6666-4666-8666-666666666666","RuntimeBootstrap":{"core_url":"https://core.example/api/v1","credential":"fixture-only","device_id":"55555555-5555-4555-8555-555555555555","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","suspend_control":false,"version":3,"workspace":"/environment/workspace"},"SessionID":"44444444-4444-4444-8444-444444444444","TenantID":"11111111-1111-4111-8111-111111111111","Workspace":{}},"valid":false}, {"kind":"request","name":"command","payload":{"Version":1,"Operation":"command","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"command_compute","payload":{"Version":1,"Operation":"command_compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Command":{"Args":["true"],"Directory":"","Stdin":null},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, {"kind":"request","name":"compute","payload":{"Version":1,"Operation":"compute","Config":{"Binary":"","StateDir":"","InstallationID":"66666666-6666-4666-8666-666666666666","APIKey":"","Template":"","APIURL":"","Domain":"","TimeoutSeconds":0,"Resources":null,"Compatibility":{"artifact_domain":"","execution_class":""}},"Reference":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333"},"Bootstrap":{"TenantID":"11111111-1111-4111-8111-111111111111","EnvironmentID":"22222222-2222-4222-8222-222222222222","AllocationID":"33333333-3333-4333-8333-333333333333","SessionID":"44444444-4444-4444-8444-444444444444","DeviceID":"55555555-5555-4555-8555-555555555555","CoreURL":"https://core.example/api/v1","Credential":"fixture-only","SuspendControl":false},"RuntimeBootstrap":{"version":3,"core_url":"https://core.example/api/v1","device_id":"55555555-5555-4555-8555-555555555555","credential":"fixture-only","environment_id":"22222222-2222-4222-8222-222222222222","session_id":"44444444-4444-4444-8444-444444444444","workspace":"/environment/workspace","suspend_control":false},"Compute":{"Generation":0,"Name":"fixture","ID":"fixture-id","RestoredFrom":null},"Deadline":"2099-01-01T00:00:00Z"},"valid":true}, From e39383b23801268c55f213c489f49fb816fce1fc Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Fri, 23 Oct 2026 21:34:27 +0000 Subject: [PATCH 05/12] Unify renewal, gate capacity help on mode, and count running sandboxes without suspended ones --- apps/web/e2e/nodes.spec.ts | 2 +- apps/web/e2e/sandbox-generation.spec.ts | 2 +- apps/web/e2e/sandbox-reset.spec.ts | 22 +++++++++---------- apps/web/e2e/sandbox-rollout-details.spec.ts | 2 +- apps/web/e2e/unified-settings.spec.ts | 2 +- .../fleet/SandboxResetNotice.test.tsx | 2 +- .../src/features/fleet/fleet-queries.test.ts | 2 +- .../features/fleet/use-sandbox-fleet.test.tsx | 4 ++-- .../src/features/overview/OverviewPage.tsx | 2 +- .../features/overview/getting-started.test.ts | 2 +- .../sandbox/SandboxResetControls.test.tsx | 2 +- .../sandbox/SandboxRolloutSummary.test.tsx | 2 +- .../features/sandbox/SuspensionSettings.tsx | 2 +- .../sandbox/sandbox-deployment-write.test.ts | 2 +- .../sandbox/sandbox-page-ownership.test.tsx | 2 +- .../features/sandbox/sandbox-queries.test.ts | 2 +- .../sandbox/use-sandbox-page-state.test.tsx | 2 +- apps/web/src/i18n/locales/zh-CN/sandbox.ts | 3 ++- contracts/agents-api/core.openapi.yaml | 3 +++ .../agents-client/src/generated/core-api.ts | 3 ++- .../agents-client/src/sandbox-client.test.ts | 6 ++--- packages/agents-client/src/sandbox-client.ts | 2 +- .../internal/db/queries/sandbox_reset.sql | 1 + .../internal/db/sqlc/sandbox_reset.sql.go | 3 +++ services/core/internal/deployment/view.go | 4 +++- .../internal/execution/runtime_lifecycle.go | 2 +- .../postgres/deploymentpg/records.go | 2 +- 27 files changed, 48 insertions(+), 37 deletions(-) diff --git a/apps/web/e2e/nodes.spec.ts b/apps/web/e2e/nodes.spec.ts index 0bc1c57d8..bb0889aa6 100644 --- a/apps/web/e2e/nodes.spec.ts +++ b/apps/web/e2e/nodes.spec.ts @@ -315,7 +315,7 @@ test("shows the retained E2B build while a replacement key is checked", async ({ test("edits only the saved backend, preserving a custom size", async ({ page, request }) => { const current = { resources: { cpus: 7, memory_mib: 8192 } }; let deployment = { configuration: {}, metadata: {}, credential_configured: false, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", core_url: "https://core.example", reset: null, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 0, preparing: 0, failed: 0, unknown: 0 } }, - owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: current, specification_digest: "e".repeat(64), + owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 0, running: 0, pending: 0 }, specification: current, specification_digest: "e".repeat(64), suspension: null }; let submitted: Record | null = null; await page.route("**/core/v1/sandbox/deployment", async (route) => { diff --git a/apps/web/e2e/sandbox-generation.spec.ts b/apps/web/e2e/sandbox-generation.spec.ts index 612823d57..a94ac1432 100644 --- a/apps/web/e2e/sandbox-generation.spec.ts +++ b/apps/web/e2e/sandbox-generation.spec.ts @@ -58,7 +58,7 @@ test("online configuration retains existing resources and stops rapid polling wh await expect(rollout(page)).toBeVisible(); const before = await inventory(page); expect(before.allocations.length).toBeGreaterThan(0); - await setDeployment(request, { resources: { allocations: before.allocations.length, pending: 0 } }); + await setDeployment(request, { resources: { allocations: before.allocations.length, running: before.allocations.length, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); await page.getByRole("button", { name: "Change resources", exact: true }).click(); await page.getByRole("button", { name: /^Large/ }).click(); diff --git a/apps/web/e2e/sandbox-reset.spec.ts b/apps/web/e2e/sandbox-reset.spec.ts index cb6c1e802..ca6cd7086 100644 --- a/apps/web/e2e/sandbox-reset.spec.ts +++ b/apps/web/e2e/sandbox-reset.spec.ts @@ -41,7 +41,7 @@ test("auto reset requires a bounded deadline, then preserves Core progress past await page.setViewportSize({ width: 1280, height: 1000 }); await page.emulateMedia({ colorScheme: "light", reducedMotion: "reduce" }); await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 3, pending: 1 } }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); await expect(dialog).toBeVisible(); @@ -71,7 +71,7 @@ test("auto reset requires a bounded deadline, then preserves Core progress past expect((await submitted).postDataJSON()).toEqual({ expected_generation: 1, clear: "auto", deadline_seconds: 300 }); await expect(progress(page)).toBeVisible(); // These timestamps are in the past: the browser still cannot invent force or completion. - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); await expect(progress(page)).toContainText("offline-build-worker"); await expect(progress(page)).toContainText("Busy work can finish until the deadline."); @@ -83,7 +83,7 @@ test("auto reset requires a bounded deadline, then preserves Core progress past test("escalation and cancellation require explicit confirmation and never undo previous clearing", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.reload(); await progress(page).getByRole("button", { name: "Force reset now" }).click(); let confirm = page.getByRole("dialog", { name: "Force reset now?" }); @@ -110,7 +110,7 @@ test("escalation and cancellation require explicit confirmation and never undo p test("force reset requires confirmation and reconfiguration uses the completed generation once", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); await dialog.getByLabel("Force — cancel remaining work now", { exact: true }).check(); @@ -134,7 +134,7 @@ test("force reset requires confirmation and reconfiguration uses the completed g test("an applied reset with a lost response stays blocked through failed reads and is never replayed", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); let failReads = false; @@ -165,7 +165,7 @@ test("an applied reset with a lost response stays blocked through failed reads a test("a stale reset is refused until fresh state is reviewed, with no automatic resubmission", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); - await setDeployment(request, { generation: 2, resources: { allocations: 3, pending: 1 } }); + await setDeployment(request, { generation: 2, resources: { allocations: 3, running: 3, pending: 1 } }); const dialog = await openReset(page); const staleSubmission = page.waitForRequest((sent) => sent.method() === "POST" && sent.url().endsWith(resetPath)); const freshRead = page.waitForResponse(async (received) => received.request().method() === "GET" @@ -195,7 +195,7 @@ test("reset progress survives failed node and deployment reads with visible qual await page.setViewportSize({ width: 1280, height: 1000 }); await page.emulateMedia({ colorScheme: "dark" }); await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.route("**/core/v1/sandbox/nodes", (route) => route.fulfill(unavailable)); await page.reload(); await expect(progress(page)).toContainText("offline-build-worker"); @@ -219,7 +219,7 @@ test("reset progress survives failed node and deployment reads with visible qual test("a pending reset survives navigation and a lost response cannot reopen stale write controls", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); let accepted!: () => void; @@ -281,7 +281,7 @@ test("Nodes and configuration adopt reset and completion learned on Overview bef await expect(page.getByRole("button", { name: "Reset deployment", exact: true })).toBeEnabled(); const cachedAt = Date.now(); await page.getByRole("button", { name: "Overview", exact: true }).click(); - await setDeployment(request, { resources: { allocations: 3, pending: 1 }, reset: observedReset() }); + await setDeployment(request, { resources: { allocations: 3, running: 3, pending: 1 }, reset: observedReset() }); await page.getByRole("button", { name: "Refresh", exact: true }).click(); await expect(page.getByRole("heading", { name: "Sandbox reset in progress" })).toBeVisible(); await expectEnrollmentBlocked(page); @@ -292,7 +292,7 @@ test("Nodes and configuration adopt reset and completion learned on Overview bef // No refresh click: shared active reset starts bounded read-only polling on configuration. const forced = observedReset("force"); forced.remaining = { busy: 0, idle: 0, cleanup: 1, on_offline_nodes: 1, offline_nodes: [{ node_id: "offline-owned", name: "offline-build-worker", resources: 1 }] }; - await setDeployment(request, { resources: { allocations: 1, pending: 0 }, reset: forced }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 }, reset: forced }); await expect(progress(page)).toContainText("Remaining hosted Sessions are being archived", { timeout: 12_000 }); await expect(progress(page).getByRole("button", { name: "Force reset now" })).toHaveCount(0); @@ -314,7 +314,7 @@ test("Nodes and configuration adopt reset and completion learned on Overview bef test("signing out clears pending reset ownership and a late response cannot overwrite the new login", async ({ page, request }) => { await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 1, pending: 0 } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 } }); await page.getByRole("button", { name: "Refresh sandbox state", exact: true }).click(); const dialog = await openReset(page); let accepted!: () => void; diff --git a/apps/web/e2e/sandbox-rollout-details.spec.ts b/apps/web/e2e/sandbox-rollout-details.spec.ts index 4695ac5ee..43d5d2ec7 100644 --- a/apps/web/e2e/sandbox-rollout-details.spec.ts +++ b/apps/web/e2e/sandbox-rollout-details.spec.ts @@ -9,7 +9,7 @@ test("keeps rollout summary brief and opens all Core observations in details", a await page.route("**/core/v1/sandbox/deployment", async (route) => { const response = await route.fetch(); const deployment = await response.json(); - await route.fulfill({ response, json: { ...deployment, generation: 4, resources: { allocations: 8, pending: 0 }, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } } }); + await route.fulfill({ response, json: { ...deployment, generation: 4, resources: { allocations: 8, running: 8, pending: 0 }, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } } }); }); await openConsole(page, request, "settings?id=sandbox"); const summary = page.getByRole("region", { name: "Configuration rollout", exact: true }); diff --git a/apps/web/e2e/unified-settings.spec.ts b/apps/web/e2e/unified-settings.spec.ts index 8b4e17c33..549e8495e 100644 --- a/apps/web/e2e/unified-settings.spec.ts +++ b/apps/web/e2e/unified-settings.spec.ts @@ -118,7 +118,7 @@ test("settings content width does not constrain the reset dialog backdrop", asyn test("reset progress actions stay inside settings on narrow screens", async ({ page, request }) => { await page.setViewportSize({ width: 375, height: 844 }); await openConsole(page, request, "settings?id=sandbox"); - await setDeployment(request, { resources: { allocations: 1, pending: 0 }, reset: { clear: "auto", requested_at: "2026-10-13T08:00:00Z", deadline_at: "2026-10-13T09:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }); + await setDeployment(request, { resources: { allocations: 1, running: 1, pending: 0 }, reset: { clear: "auto", requested_at: "2026-10-13T08:00:00Z", deadline_at: "2026-10-13T09:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }); await page.getByRole("button", { name: "Refresh sandbox state" }).click(); const region = page.getByRole("region", { name: "Reset in progress" }); await expect(region).toBeVisible(); diff --git a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx index 0cb7598a1..79341708d 100644 --- a/apps/web/src/features/fleet/SandboxResetNotice.test.tsx +++ b/apps/web/src/features/fleet/SandboxResetNotice.test.tsx @@ -6,7 +6,7 @@ import { SandboxResetNotice } from "./SandboxResetNotice"; const reset: SandboxReset = { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 2, on_offline_nodes: 1, offline_nodes: [{ node_id: "n1", name: "Node 1", resources: 1 }] } }; -const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, pending: 0 }, suspension: null }); +const deployment = (reset: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 3, running: 3, pending: 0 }, suspension: null }); const render = (value: SandboxDeployment | undefined, failed = false) => renderToStaticMarkup( {}} />); describe("reset notices on read-only surfaces", () => { diff --git a/apps/web/src/features/fleet/fleet-queries.test.ts b/apps/web/src/features/fleet/fleet-queries.test.ts index e0eb11cc1..aabdcc5c4 100644 --- a/apps/web/src/features/fleet/fleet-queries.test.ts +++ b/apps/web/src/features/fleet/fleet-queries.test.ts @@ -5,7 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxScope } from "../sandbox/sandbox-queries"; import { fleetQuery } from "./fleet-queries"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null, +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, running: 1, pending: 0 }, suspension: null, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); }); diff --git a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx index 756ae7660..49a8ba840 100644 --- a/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx +++ b/apps/web/src/features/fleet/use-sandbox-fleet.test.tsx @@ -10,7 +10,7 @@ import { fleetQuery, type FleetSnapshot } from "./fleet-queries"; import { FleetReadNotice } from "./FleetReadNotice"; import { fleetSnapshot, useSandboxFleet } from "./use-sandbox-fleet"; -const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, running: 1, pending: 0 }, suspension: null }; function Probe() { const { state, deployment } = useSandboxFleet(); @@ -33,7 +33,7 @@ function render(latest: SandboxDeployment, previous = configured, failed = false describe("fleet evidence after deployment changes", () => { it("cannot complete onboarding or show old resource counts when reset leaves an unconfigured deployment", () => { const resetting = { ...configured, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } } satisfies SandboxDeployment; - const completed = { ...configured, provider: "", mode: "", generation: 2, resources: { allocations: 0, pending: 0 } } satisfies SandboxDeployment; + const completed = { ...configured, provider: "", mode: "", generation: 2, resources: { allocations: 0, running: 0, pending: 0 } } satisfies SandboxDeployment; const html = render(completed, resetting); expect(html).toContain('data-fleet="loading"'); expect(html).toContain('data-sandbox="checking"'); diff --git a/apps/web/src/features/overview/OverviewPage.tsx b/apps/web/src/features/overview/OverviewPage.tsx index fe907ecfc..a43548929 100644 --- a/apps/web/src/features/overview/OverviewPage.tsx +++ b/apps/web/src/features/overview/OverviewPage.tsx @@ -316,7 +316,7 @@ function fleetDetail(state: FleetState, t: TFunction<"overview">): string { function cloudHost(fleet: FleetSnapshot | null): CloudHost | null { if (fleet?.deployment.mode !== "direct") return null; - return { running: fleet.deployment.resources.allocations, pending: fleet.deployment.resources.pending }; + return { running: fleet.deployment.resources.running, pending: fleet.deployment.resources.pending }; } /** Core and its sandbox nodes (or the Provider's cloud) as an inventory; each row opens its page. */ diff --git a/apps/web/src/features/overview/getting-started.test.ts b/apps/web/src/features/overview/getting-started.test.ts index 0409d1a5f..aaffd5da0 100644 --- a/apps/web/src/features/overview/getting-started.test.ts +++ b/apps/web/src/features/overview/getting-started.test.ts @@ -7,7 +7,7 @@ import { node, project } from "./test-fixtures"; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", - resources: { allocations: 0, pending: 0 }, suspension: null, ...overrides, + resources: { allocations: 0, running: 0, pending: 0 }, suspension: null, ...overrides, }); const fleet = (value: SandboxDeployment, nodes = [node("n1")]): FleetState => ({ status: "ready", snapshot: { deployment: value, nodes, allocations: [], loadedAt: 0 }, targetGeneration: value.generation, refreshing: false, error: null }); const sandboxes = (state: FleetState) => gettingStartedSteps({ sandboxReset: false, fleet: state, localOnly: false, projects: [], sessions: 0, harnesses: [] }).sandboxes; diff --git a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx index c5184de13..f8678adae 100644 --- a/apps/web/src/features/sandbox/SandboxResetControls.test.tsx +++ b/apps/web/src/features/sandbox/SandboxResetControls.test.tsx @@ -5,7 +5,7 @@ import { SandboxResetControls } from "./SandboxResetControls"; const reset: SandboxReset = { clear: "auto", requested_at: "2020-01-01T10:00:00Z", deadline_at: "2020-01-01T11:00:00Z", forced_at: null, remaining: { busy: 2, idle: 1, cleanup: 3, on_offline_nodes: 2, offline_nodes: [{ node_id: "offline-a", name: "Offline A", resources: 2 }] } }; -const deployment = (value: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, pending: 1 }, suspension: null }); +const deployment = (value: SandboxReset | null): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "https://core.example", reset: value, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 5, running: 5, pending: 1 }, suspension: null }); const render = (value: SandboxReset | null, stale = false) => renderToStaticMarkup( true} onCancel={async () => true} />); describe("authoritative reset progress", () => { diff --git a/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx b/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx index 0364ada21..732cc1cf7 100644 --- a/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx +++ b/apps/web/src/features/sandbox/SandboxRolloutSummary.test.tsx @@ -7,7 +7,7 @@ import { SandboxRolloutSummary } from "./SandboxRolloutSummary"; import { NodeRolloutStatus } from "./NodeRolloutStatus"; import { SandboxDeploymentSettings } from "./SandboxDeploymentSettings"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, installation_id: "i", owner_epoch: 1, generation: 4, provider: "docker", mode: "nodes", core_url: "https://core.example", resources: { allocations: 8, pending: 2 }, reset: null, suspension: null, +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, installation_id: "i", owner_epoch: 1, generation: 4, provider: "docker", mode: "nodes", core_url: "https://core.example", resources: { allocations: 8, running: 8, pending: 2 }, reset: null, suspension: null, rollout: { state: "settled", previous_generation_sandboxes: 8, nodes: { ready: 1, preparing: 0, failed: 2, unknown: 4 } } }; describe("authoritative configuration rollout", () => { diff --git a/apps/web/src/features/sandbox/SuspensionSettings.tsx b/apps/web/src/features/sandbox/SuspensionSettings.tsx index 99fb1f6fd..15232b677 100644 --- a/apps/web/src/features/sandbox/SuspensionSettings.tsx +++ b/apps/web/src/features/sandbox/SuspensionSettings.tsx @@ -44,7 +44,7 @@ export function SuspensionForm({ deployment, disabled, onSave }: { deployment: S return
{ event.preventDefault(); if (valid && !disabled && await onSave(Number(idle))) setDraft(null); }}> -
{t("Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged. Capacity pressure may suspend an idle sandbox after 15 seconds.")}
+
{t("Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged.")}{deployment.mode === "direct" ? null : <> {t("Capacity pressure may suspend an idle sandbox after 15 seconds.")}}
setDraft(event.target.value)} aria-invalid={!valid} aria-describedby={!valid ? "sandbox-idle-error" : undefined} /> diff --git a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts index be04e2036..d1066ddd3 100644 --- a/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts +++ b/apps/web/src/features/sandbox/sandbox-deployment-write.test.ts @@ -5,7 +5,7 @@ import { sandboxAdmin, sandboxDeploymentQuery } from "./sandbox-queries"; import { writeSandboxDeployment } from "./sandbox-deployment-write"; import { sandboxWriteOwnershipQuery } from "./sandbox-write-ownership"; -const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null }; +const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", owner_epoch: 1, generation: 1, provider: "docker", core_url: "http://core", mode: "nodes", reset: null, resources: { allocations: 0, running: 0, pending: 0 }, suspension: null }; function deferred() { let resolve!: (value: T) => void; let reject!: (error: Error) => void; diff --git a/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx b/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx index d9842bc34..dddbf62b0 100644 --- a/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx +++ b/apps/web/src/features/sandbox/sandbox-page-ownership.test.tsx @@ -18,7 +18,7 @@ function cache(provider: SandboxDeployment["provider"]) { const client = new QueryClient({ defaultOptions: { queries: { enabled: false, retry: false, gcTime: Infinity } } }); const deployment: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, installation_id: "install", owner_epoch: 1, generation: 2, provider, core_url: "https://core.example", - mode: provider === "" ? "" : provider === "e2b" ? "direct" : "nodes", reset: null, resources: { allocations: 0, pending: 0 }, suspension: null, + mode: provider === "" ? "" : provider === "e2b" ? "direct" : "nodes", reset: null, resources: { allocations: 0, running: 0, pending: 0 }, suspension: null, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, }; const config: SandboxConsoleConfig = { node_installer: false, node_installer_sha256: "", node_artifacts: [] }; diff --git a/apps/web/src/features/sandbox/sandbox-queries.test.ts b/apps/web/src/features/sandbox/sandbox-queries.test.ts index 62bef4afb..08854805d 100644 --- a/apps/web/src/features/sandbox/sandbox-queries.test.ts +++ b/apps/web/src/features/sandbox/sandbox-queries.test.ts @@ -10,7 +10,7 @@ const reset: SandboxReset = { }; const deployment = (overrides: Partial = {}): SandboxDeployment => ({ credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "installation-a", provider: "docker", core_url: "https://core.example", reset, - owner_epoch: 1, generation: 2, mode: "nodes", resources: { allocations: 2, pending: 0 }, suspension: null, ...overrides, + owner_epoch: 1, generation: 2, mode: "nodes", resources: { allocations: 2, running: 2, pending: 0 }, suspension: null, ...overrides, }); const clients: QueryClient[] = []; const cache = () => { const client = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: Infinity } } }); clients.push(client); return client; }; diff --git a/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx b/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx index 63c8667c2..ae16d744a 100644 --- a/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx +++ b/apps/web/src/features/sandbox/use-sandbox-page-state.test.tsx @@ -6,7 +6,7 @@ import { node } from "../overview/test-fixtures"; import { sandboxAdmin, sandboxDeploymentQuery, sandboxSnapshotQuery } from "./sandbox-queries"; import { useSandboxPageState } from "./use-sandbox-page-state"; -const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, pending: 0 }, suspension: null }; +const configured: SandboxDeployment = { credential_configured: false, configuration: {}, metadata: {}, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, installation_id: "i", provider: "docker", core_url: "http://core", reset: null, owner_epoch: 1, generation: 1, mode: "nodes", resources: { allocations: 1, running: 1, pending: 0 }, suspension: null }; const resetting: SandboxDeployment = { ...configured, reset: { clear: "auto", requested_at: "2026-09-27T10:00:00Z", deadline_at: "2026-09-27T11:00:00Z", forced_at: null, remaining: { busy: 1, idle: 0, cleanup: 0, on_offline_nodes: 0, offline_nodes: [] } } }; const clients: QueryClient[] = []; function cache() { diff --git a/apps/web/src/i18n/locales/zh-CN/sandbox.ts b/apps/web/src/i18n/locales/zh-CN/sandbox.ts index 8feac385c..116629d96 100644 --- a/apps/web/src/i18n/locales/zh-CN/sandbox.ts +++ b/apps/web/src/i18n/locales/zh-CN/sandbox.ts @@ -5,7 +5,8 @@ export const sandbox = { "Reset": "重置", "Clear this configuration to choose a different backend. Hosted Sessions are archived permanently.": "清除当前配置后才能选择其他沙箱提供商。托管 Session 将被永久归档。", "Idle before suspension (seconds)": "空闲多久后挂起（秒）", - "Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged. Capacity pressure may suspend an idle sandbox after 15 seconds.": "整个部署统一生效：30–86,400 秒。适用于现有和新建的运行中沙箱，无需重启。已经开始的挂起和快照保留期不变。容量紧张时，沙箱空闲 15 秒后可能提前挂起。", + "Deployment-wide: 30–86,400 seconds. Applies to existing and new running sandboxes without restarting them. Suspension already in progress and snapshot retention are unchanged.": "整个部署统一生效：30–86,400 秒。适用于现有和新建的运行中沙箱，无需重启。已经开始的挂起和快照保留期不变。", + "Capacity pressure may suspend an idle sandbox after 15 seconds.": "容量紧张时，沙箱空闲 15 秒后可能提前挂起。", "Enter a whole number from 30 to 86,400.": "请输入 30 到 86,400 之间的整数。", "Automatic suspension": "自动挂起", "Current policy for new suspensions, not a countdown for existing snapshots. The next Turn requests a resume.": "这是新挂起任务采用的当前策略，不是现有快照的到期倒计时。下一轮 Turn 会请求恢复。", diff --git a/contracts/agents-api/core.openapi.yaml b/contracts/agents-api/core.openapi.yaml index e0d305ff2..967570464 100644 --- a/contracts/agents-api/core.openapi.yaml +++ b/contracts/agents-api/core.openapi.yaml @@ -1637,9 +1637,12 @@ definitions: type: integer pending: type: integer + running: + type: integer required: - allocations - pending + - running type: object deployment.Rollout: properties: diff --git a/packages/agents-client/src/generated/core-api.ts b/packages/agents-client/src/generated/core-api.ts index b8eeb6163..074141c2f 100644 --- a/packages/agents-client/src/generated/core-api.ts +++ b/packages/agents-client/src/generated/core-api.ts @@ -209,8 +209,9 @@ export type DeploymentMode = (typeof deploymentModeValues)[number]; export interface DeploymentResources { allocations: number; pending: number; + running: number; } -export const deploymentResourcesFields = ["allocations", "pending"] as const; +export const deploymentResourcesFields = ["allocations", "pending", "running"] as const; export interface DeploymentSpec { resources: SandboxResources; workspace?: Declaration; diff --git a/packages/agents-client/src/sandbox-client.test.ts b/packages/agents-client/src/sandbox-client.test.ts index 126533cf9..b944851d4 100644 --- a/packages/agents-client/src/sandbox-client.test.ts +++ b/packages/agents-client/src/sandbox-client.test.ts @@ -44,11 +44,11 @@ const allocation = { state: "running", compute_phase: "running", compute_phase_changed_at: null, diagnostic: "", initialization: "ready", created_at: created, }; const unconfigured = { - credential_configured: false, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; + credential_configured: false, rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: null }, installation_id: "", provider: "", core_url: "https://core.example", reset: null, owner_epoch: 0, generation: 0, mode: "", resources: { allocations: 0, running: 0, pending: 0 }, suspension: null }; const docker = { configuration: {}, metadata: {}, ...unconfigured, installation_id: "94be54a1-138c-4f30-bc87-b13686272dbe", provider: "docker", rollout: { state: "settled", previous_generation_sandboxes: 0, nodes: { ready: 1, preparing: 0, failed: 0, unknown: 0 } }, owner_epoch: 1, generation: 1, mode: "nodes", - resources: { allocations: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, specification_digest: "0".repeat(64), + resources: { allocations: 2, running: 2, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, specification_digest: "0".repeat(64), }; const microsandbox = { ...docker, provider: "microsandbox", specification: { resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 8192 } }, @@ -254,7 +254,7 @@ describe("hosted provider configuration", () => { const e2b = { api_key: "test-only-secret", template: "runtime:00000000-0000-0000-0000-000000000001" }; it("writes E2B configuration and generation without beta headers or browser credentials", async () => { const deployment = { - ...e2bDeployment, generation: 2, resources: { allocations: 0, pending: 0 }, + ...e2bDeployment, generation: 2, resources: { allocations: 0, running: 0, pending: 0 }, configuration: { template: e2b.template, api_url: "https://api.e2b.app", domain: "e2b.app" } , credential_configured: true, metadata: { template_build: { status: "ready", resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 24063 } } }, }; const fetch = vi.fn().mockImplementation(async () => response(deployment)); diff --git a/packages/agents-client/src/sandbox-client.ts b/packages/agents-client/src/sandbox-client.ts index 2af93b40f..578760864 100644 --- a/packages/agents-client/src/sandbox-client.ts +++ b/packages/agents-client/src/sandbox-client.ts @@ -160,7 +160,7 @@ function projectDeployment(value: unknown): SandboxDeployment { const suspension = deployment.suspension === null ? null : members(deployment.suspension, suspensionFields); const configured = hasOwn(deployment, "specification"); valid(strings(deployment, ["installation_id", "core_url"]) && (deployment.provider === "" || (typeof deployment.provider === "string" && hasOwn(deploymentContract.providers, deployment.provider))) && isOneOf(deploymentModeValues, deployment.mode) && - [deployment.owner_epoch, deployment.generation, resources.allocations, resources.pending].every(isNonnegativeInteger) && + [deployment.owner_epoch, deployment.generation, resources.allocations, resources.running, resources.pending].every(isNonnegativeInteger) && (suspension === null || [suspension.idle_seconds, suspension.retention_seconds].every(isNonnegativeInteger)) && configured === hasOwn(deployment, "specification_digest") && (!configured || (typeof deployment.specification_digest === "string" && deployment.specification_digest !== ""))); return { diff --git a/services/core/internal/db/queries/sandbox_reset.sql b/services/core/internal/db/queries/sandbox_reset.sql index 335321ac4..cbadae68d 100644 --- a/services/core/internal/db/queries/sandbox_reset.sql +++ b/services/core/internal/db/queries/sandbox_reset.sql @@ -100,6 +100,7 @@ held AS ( SELECT sqlc.embed(d), (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, + (SELECT count(*) FROM runtime_allocations a WHERE a.state = 'running' AND a.compute_phase IN ('running', 'disabled'))::bigint AS running, jsonb_build_object( 'busy', (SELECT count(*) FROM classified WHERE category = 'busy'), 'idle', (SELECT count(*) FROM classified WHERE category = 'idle'), diff --git a/services/core/internal/db/sqlc/sandbox_reset.sql.go b/services/core/internal/db/sqlc/sandbox_reset.sql.go index 302fe1620..4412e8214 100644 --- a/services/core/internal/db/sqlc/sandbox_reset.sql.go +++ b/services/core/internal/db/sqlc/sandbox_reset.sql.go @@ -93,6 +93,7 @@ held AS ( SELECT d.singleton, d.installation_id, d.backend_fingerprint, d.updated_at, d.provider_kind, d.owner_epoch, d.generation, d.mode, d.provider_credential, d.specification, d.reset_clear, d.reset_requested_at, d.reset_deadline_at, d.reset_forced_at, d.reset_audit, d.provider_config, d.provider_metadata, d.idle_seconds, (SELECT count(*) FROM classified WHERE NOT pending)::bigint AS allocations, (SELECT count(*) FROM classified WHERE pending)::bigint AS pending, + (SELECT count(*) FROM runtime_allocations a WHERE a.state = 'running' AND a.compute_phase IN ('running', 'disabled'))::bigint AS running, jsonb_build_object( 'busy', (SELECT count(*) FROM classified WHERE category = 'busy'), 'idle', (SELECT count(*) FROM classified WHERE category = 'idle'), @@ -117,6 +118,7 @@ type GetSandboxDeploymentSnapshotRow struct { RuntimeDeployment RuntimeDeployment `json:"runtime_deployment"` Allocations int64 `json:"allocations"` Pending int64 `json:"pending"` + Running int64 `json:"running"` Remaining []byte `json:"remaining"` Rollout []byte `json:"rollout"` } @@ -145,6 +147,7 @@ func (q *Queries) GetSandboxDeploymentSnapshot(ctx context.Context) (GetSandboxD &i.RuntimeDeployment.IdleSeconds, &i.Allocations, &i.Pending, + &i.Running, &i.Remaining, &i.Rollout, ) diff --git a/services/core/internal/deployment/view.go b/services/core/internal/deployment/view.go index 88cf94ef7..253fa0da0 100644 --- a/services/core/internal/deployment/view.go +++ b/services/core/internal/deployment/view.go @@ -44,9 +44,11 @@ type View struct { CoreURL string `json:"core_url" binding:"required"` } -// Resources counts what still belongs to the deployment. +// Resources counts what still belongs to the deployment. Running excludes +// suspended allocations, which Allocations includes. type Resources struct { Allocations int64 `json:"allocations" binding:"required"` + Running int64 `json:"running" binding:"required"` Pending int64 `json:"pending" binding:"required"` } diff --git a/services/core/internal/execution/runtime_lifecycle.go b/services/core/internal/execution/runtime_lifecycle.go index ceb23fd40..12f5a7cd4 100644 --- a/services/core/internal/execution/runtime_lifecycle.go +++ b/services/core/internal/execution/runtime_lifecycle.go @@ -467,7 +467,7 @@ func (r *runtimeLifecycle) observe(ctx context.Context, owner deployment.Allocat if r.config.Suspension != nil && environment.Initialization == "complete" { return r.enableCompute(ctx, owner) } - if peer, err := r.registry.LookupDevice(owner.DeviceID); err != nil || peer.IsClosed() { + if _, err := authorizedRuntimePeer(ctx, r.sessions, r.registry, owner.DeviceID); err != nil { return nil } renewed, err := provider.Renew(ctx, runtimeReference(owner)) diff --git a/services/core/internal/persistence/postgres/deploymentpg/records.go b/services/core/internal/persistence/postgres/deploymentpg/records.go index a17f2a5a9..ea89846a9 100644 --- a/services/core/internal/persistence/postgres/deploymentpg/records.go +++ b/services/core/internal/persistence/postgres/deploymentpg/records.go @@ -75,7 +75,7 @@ func record(d sqlc.RuntimeDeployment, cipher *credentialcrypto.Cipher, open bool } func snapshot(row sqlc.GetSandboxDeploymentSnapshotRow) (deployment.Snapshot, error) { - result := deployment.Snapshot{Record: record(row.RuntimeDeployment, nil, false), Resources: deployment.Resources{Allocations: row.Allocations, Pending: row.Pending}} + result := deployment.Snapshot{Record: record(row.RuntimeDeployment, nil, false), Resources: deployment.Resources{Allocations: row.Allocations, Running: row.Running, Pending: row.Pending}} if err := json.Unmarshal(row.Rollout, &result.Rollout); err != nil { return deployment.Snapshot{}, err } From eaeca4b8041fd595a7ec7c0372c021977012d30f Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Fri, 23 Oct 2026 21:36:23 +0000 Subject: [PATCH 06/12] Refresh the sandbox provider translation hash --- docs/zh/sandbox-provider.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/sandbox-provider.md b/docs/zh/sandbox-provider.md index 67427f47c..4e3b93061 100644 --- a/docs/zh/sandbox-provider.md +++ b/docs/zh/sandbox-provider.md @@ -1,7 +1,7 @@ --- title: "沙箱提供商协议" source: docs/sandbox-provider.md -source_hash: 199dc6de344085e9b8c1048e7e8808aa40353878865061236e1a919ad689e28b +source_hash: 8685196568aa5999018e746979b6c85f59873e6fe520ce14e73f91d7a23facff --- 沙箱提供商协议连接 Core 与为 Core 管理的（`openai_hosted`）Environment 提供计算资源的 adapter，并由该 adapter 运行启动 Runtime daemon 的有界引导流程。本页面向 adapter 作者：说明提供商必须实现什么，以及可以依赖 Core 做什么。Docker、microsandbox 和 E2B 实现了该协议。 From 9411132d59776e00ae39ae4806fd7c9c137a867b Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sat, 24 Oct 2026 01:25:04 +0000 Subject: [PATCH 07/12] Declare checkpoint support in the execution test Providers --- .../internal/execution/runtime_pending_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/services/core/internal/execution/runtime_pending_test.go b/services/core/internal/execution/runtime_pending_test.go index 575d32667..0d8063e2c 100644 --- a/services/core/internal/execution/runtime_pending_test.go +++ b/services/core/internal/execution/runtime_pending_test.go @@ -3,6 +3,7 @@ package execution import ( "context" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/providercontract" "net/http" "net/http/httptest" "net/url" @@ -381,3 +382,19 @@ func TestAwaitingConnectionPassSharesCancellationFence(t *testing.T) { t.Fatal("connection pass escaped cancellation") } } + +// noCheckpoint declares a fixture Provider that supports no checkpoint. +func noCheckpoint() providercontract.Operations { + return providercontract.Operations{"Initial": {State: providercontract.Unsupported, Reason: "fixture_operation_not_supported"}} +} + +func (*connectedBootstrapProvider) ProviderOperations() providercontract.Operations { + return noCheckpoint() +} +func (*settledWorkspaceRefusal) ProviderOperations() providercontract.Operations { + return noCheckpoint() +} +func (*unknownWorkspaceCreation) ProviderOperations() providercontract.Operations { + return noCheckpoint() +} +func (*preparationProvider) ProviderOperations() providercontract.Operations { return noCheckpoint() } From 67551d64406b7bf008562487110b4c7d736c7f63 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sat, 24 Oct 2026 01:26:46 +0000 Subject: [PATCH 08/12] Exclude suspend control from the managed bootstrap projection check --- services/core/internal/sandbox/e2b/helper_contract_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/core/internal/sandbox/e2b/helper_contract_test.go b/services/core/internal/sandbox/e2b/helper_contract_test.go index a40130f89..76c2dcd18 100644 --- a/services/core/internal/sandbox/e2b/helper_contract_test.go +++ b/services/core/internal/sandbox/e2b/helper_contract_test.go @@ -66,6 +66,7 @@ func validManagedExchange(data []byte) bool { _ = json.Unmarshal(bootstrapBytes, &expected) delete(expected, "CoreURL") delete(expected, "Credential") + delete(expected, "SuspendControl") expected["InstallationID"] = nil expected["RuntimeBootstrap"] = nil if workspace, present := fields["Workspace"]; present { From 11abebcee4ce9000c9957dfe2166596591125128 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sat, 24 Oct 2026 01:29:45 +0000 Subject: [PATCH 09/12] Fail a stale E2B template at create --- services/core/deploy/e2b/README.md | 2 +- .../deploy/e2b/helper_contract_generated.py | 1 + .../sandbox/e2b/internal/contractgen/main.go | 2 +- .../core/internal/sandbox/e2b/provider.go | 2 +- .../e2b-provider/helper_contract_generated.py | 1 + services/core/tools/e2b-provider/provider.py | 21 +++++++++++++++---- .../core/tools/e2b-provider/provider_test.py | 18 ++++++++++++++++ 7 files changed, 40 insertions(+), 7 deletions(-) diff --git a/services/core/deploy/e2b/README.md b/services/core/deploy/e2b/README.md index 93a7d6c55..bb244e9f7 100644 --- a/services/core/deploy/e2b/README.md +++ b/services/core/deploy/e2b/README.md @@ -21,7 +21,7 @@ python -m venv "$HOME/.oac/build/e2b-sdk" --output "$HOME/.oac/build/e2b-template.json" ``` -[`build-template.py`](build-template.py) requires a `sha256:` image ID, a Linux amd64 image and the Runtime layout. It copies the image's `/usr/local/bin`, `/opt` and, when present, `/usr/local/codex-resources` and `/etc/codex`, together with its `HOME` and `OAC_*` environment, onto a digest-pinned `node:22.23.1-bookworm-slim` base with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip`, `ripgrep` and `util-linux`. It installs [`init.py`](init.py) and [`managed_init.py`](managed_init.py) read-only under `/opt/oac-e2b`, runs the daemon as UID/GID 1000 (`runtime`, home `/home/runtime`) and builds with 2 vCPUs and 2048 MiB. Only the `usr`, `usr/local` and `etc` archive ancestors it creates get traversable modes; Runtime file modes, private build contexts, key inputs and the output's umask stay unchanged. +[`build-template.py`](build-template.py) requires a `sha256:` image ID, a Linux amd64 image and the Runtime layout. It copies the image's `/usr/local/bin`, `/opt` and, when present, `/usr/local/codex-resources` and `/etc/codex`, together with its `HOME` and `OAC_*` environment, onto a digest-pinned `node:22.23.1-bookworm-slim` base with `ca-certificates`, `bash`, `git`, `python3`, `python3-pip`, `ripgrep` and `util-linux`. Rebuild a template whenever the Runtime bootstrap version changes: Create refuses a template whose contract names another version. It installs [`init.py`](init.py) and [`managed_init.py`](managed_init.py) read-only under `/opt/oac-e2b`, runs the daemon as UID/GID 1000 (`runtime`, home `/home/runtime`) and builds with 2 vCPUs and 2048 MiB. Only the `usr`, `usr/local` and `etc` archive ancestors it creates get traversable modes; Runtime file modes, private build contexts, key inputs and the output's umask stay unchanged. The output file records `template` (the immutable `templateID:build_UUID`), the source `image`, the packaged `runtime_sha256` and the `base`. Use that exact `template` value. The template must qualify every Harness its image advertises. Install system dependencies into the image; the daemon runs as UID/GID 1000. No E2B account key, executor key or model credential belongs in a build, template environment, metadata, command argument or log. diff --git a/services/core/deploy/e2b/helper_contract_generated.py b/services/core/deploy/e2b/helper_contract_generated.py index bf7f1aa4a..bee1ee648 100644 --- a/services/core/deploy/e2b/helper_contract_generated.py +++ b/services/core/deploy/e2b/helper_contract_generated.py @@ -14,5 +14,6 @@ REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Snapshot","Deadline"] RESPONSE_FIELDS = ["Version","Info","Command","State","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RUNTIME_BOOTSTRAP_VERSION = 3 SDK_VERSION = "2.51.0" SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" diff --git a/services/core/internal/sandbox/e2b/internal/contractgen/main.go b/services/core/internal/sandbox/e2b/internal/contractgen/main.go index fd45fc5cc..aaa39d639 100644 --- a/services/core/internal/sandbox/e2b/internal/contractgen/main.go +++ b/services/core/internal/sandbox/e2b/internal/contractgen/main.go @@ -81,7 +81,7 @@ func main() { identity = append(identity, "InstallationID") values := map[string]any{ "PROTOCOL_VERSION": e2b.ProtocolVersion, "SDK_VERSION": sdk, - "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, + "SUSPEND_CONTROL_FILE": runtimebootstrap.SuspendControlFile, "RUNTIME_BOOTSTRAP_VERSION": runtimebootstrap.Version, "MAX_REQUEST": e2b.MaxRequestBytes, "MAX_RESPONSE": e2b.MaxResponseBytes, "MAX_OUTPUT": e2b.MaxOutputBytes, "MAX_COMMAND_INPUT": e2b.MaxCommandInputBytes, "MAX_CREDENTIAL_REFERENCES": e2b.MaxCredentialReferences, "OPERATIONS": e2b.HelperOperations(), "ERROR_CODES": e2b.HelperErrors(), "REQUEST_FIELDS": fields(reflect.TypeFor[e2b.Request]()), "RESPONSE_FIELDS": fields(reflect.TypeFor[e2b.Response]()), diff --git a/services/core/internal/sandbox/e2b/provider.go b/services/core/internal/sandbox/e2b/provider.go index 92ecc5b4b..ef4e2dc50 100644 --- a/services/core/internal/sandbox/e2b/provider.go +++ b/services/core/internal/sandbox/e2b/provider.go @@ -220,7 +220,7 @@ func (p *Provider) exchange(ctx context.Context, q Request) (Response, error) { case "": return out, nil case "template_invalid": - return out, fmt.Errorf("%w: This E2B template lacks the current Runtime startup entry point. Build a template with this release's build-template.py and select it in the sandbox deployment.", sandbox.ErrInvalid) + return out, fmt.Errorf("%w: This E2B template lacks the current Runtime startup entry point or was built for another Runtime bootstrap version. Build a template with this release's build-template.py and select it in the sandbox deployment.", sandbox.ErrInvalid) case "team_mismatch": return out, sandbox.ErrCredentialOwnership case "unauthorized": diff --git a/services/core/tools/e2b-provider/helper_contract_generated.py b/services/core/tools/e2b-provider/helper_contract_generated.py index bf7f1aa4a..bee1ee648 100644 --- a/services/core/tools/e2b-provider/helper_contract_generated.py +++ b/services/core/tools/e2b-provider/helper_contract_generated.py @@ -14,5 +14,6 @@ REFERENCE_FIELDS = ["TenantID","EnvironmentID","AllocationID"] REQUEST_FIELDS = ["Version","Operation","Config","Reference","References","Bootstrap","RuntimeBootstrap","Command","Compute","Suspend","Resume","Snapshot","Deadline"] RESPONSE_FIELDS = ["Version","Info","Command","State","ErrorCode","DeploymentValid","TemplateBuild","Templates","Builds","Observation"] +RUNTIME_BOOTSTRAP_VERSION = 3 SDK_VERSION = "2.51.0" SUSPEND_CONTROL_FILE = "/run/oac/daemon-suspend.json" diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index 082e71891..c84db9726 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -14,10 +14,23 @@ from sdk import connection_material, definitely_rejected, list_builds, list_templates, read_metrics, restore, run, sdk_options, validate_deployment, verify_team_template from state import Failure, Receipt, private_root, read_receipt from helper_contract_generated import (PROTOCOL_VERSION, OPERATIONS, REQUEST_FIELDS, REFERENCE_FIELDS, - MAX_CREDENTIAL_REFERENCES, MANAGED_BOOTSTRAP_FIELDS, MANAGED_BOOTSTRAP_REQUIRED_FIELDS) + MAX_CREDENTIAL_REFERENCES, MANAGED_BOOTSTRAP_FIELDS, MANAGED_BOOTSTRAP_REQUIRED_FIELDS, RUNTIME_BOOTSTRAP_VERSION) PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +# Exits 78 unless the template carries the startup script and a contract for this Runtime bootstrap version. +TEMPLATE_CHECK = """import importlib.util, os, sys +root, version = sys.argv[1], int(sys.argv[2]) +script = os.path.join(root, 'managed_init.py') +try: + spec = importlib.util.spec_from_file_location('template_contract', os.path.join(root, 'helper_contract_generated.py')) + contract = importlib.util.module_from_spec(spec) + spec.loader.exec_module(contract) + current = getattr(contract, 'RUNTIME_BOOTSTRAP_VERSION', None) +except Exception: + current = None +sys.exit(78 if not os.path.isfile(script) or not os.access(script, os.R_OK) or current != version else 0) +""" COMPUTE_OPERATIONS = ('compute', 'renew_compute', 'resume_compute', 'suspend', 'resume', 'kill_compute', 'delete_snapshot', 'command_compute') @@ -235,8 +248,7 @@ def create(self): self.receipt.save(status='configuration_rejected', settled=True) raise # Validate the current template entry point before writing any credential. - check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', - "import os,sys; sys.exit(78 if not os.path.isfile('/opt/oac-e2b/managed_init.py') or not os.access('/opt/oac-e2b/managed_init.py', os.R_OK) else 0)"]}, + check = run(cloud, {'Args': ['/usr/bin/python3', '-I', '-c', TEMPLATE_CHECK, '/opt/oac-e2b', str(RUNTIME_BOOTSTRAP_VERSION)]}, self.remaining, user='root') if check['ExitCode'] != 0: self.receipt.save(status='bootstrap_failed', settled=True) @@ -382,7 +394,8 @@ def suspend(self): pause = (self.receipt.data or {}).get('pause') or {} if cloud.state != 'paused' or pause.get('operation_id') != operation: # A dispatched pause may still take effect, so running is never reported for it. - if request['ObserveOnly'] and not (pause.get('operation_id') == operation and pause.get('status') == 'pending'): + pending = pause.get('operation_id') == operation and pause.get('status') == 'pending' + if request['ObserveOnly'] and not pending: return self.state(source, cloud) raise Failure('unconfirmed') self.receipt.save(pause={'operation_id': operation, 'status': 'paused'}) diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 5118fdcca..76b27870f 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -3,7 +3,10 @@ from datetime import datetime, timedelta, timezone import json from pathlib import Path +import subprocess +import sys import tempfile +import time from types import SimpleNamespace import unittest from unittest.mock import Mock, patch @@ -12,6 +15,7 @@ from e2b import SandboxState from e2b.exceptions import AuthenticationException, SandboxNotFoundException +import provider from provider import Provider from sdk import restore, run from state import Failure, Receipt @@ -324,6 +328,20 @@ def test_unknown_command_does_not_replay(self): client.commands.run.assert_called_once() +class TemplateCheckTest(unittest.TestCase): + def exit_code(self, files): + with tempfile.TemporaryDirectory() as root: + for name, text in files.items(): + Path(root, name).write_text(text) + return subprocess.run([sys.executable, '-I', '-c', provider.TEMPLATE_CHECK, root, '3']).returncode + + def test_template_needs_startup_script_and_current_bootstrap_version(self): + self.assertEqual(self.exit_code({'managed_init.py': '', 'helper_contract_generated.py': 'RUNTIME_BOOTSTRAP_VERSION = 3'}), 0) + self.assertEqual(self.exit_code({'managed_init.py': '', 'helper_contract_generated.py': 'RUNTIME_BOOTSTRAP_VERSION = 2'}), 78) + self.assertEqual(self.exit_code({'managed_init.py': '', 'helper_contract_generated.py': ''}), 78) + self.assertEqual(self.exit_code({'helper_contract_generated.py': 'RUNTIME_BOOTSTRAP_VERSION = 3'}), 78) + + class CheckpointTest(Fixture): """One sandbox spans generations: pause keeps it and connect resumes it.""" From 48297f8d87823214e3731630865d00aa56f57b77 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sat, 24 Oct 2026 01:29:45 +0000 Subject: [PATCH 10/12] Settle an old pending E2B pause by observation --- services/core/tools/e2b-provider/provider.py | 8 +++++++- services/core/tools/e2b-provider/provider_test.py | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/services/core/tools/e2b-provider/provider.py b/services/core/tools/e2b-provider/provider.py index c84db9726..3296d1967 100644 --- a/services/core/tools/e2b-provider/provider.py +++ b/services/core/tools/e2b-provider/provider.py @@ -18,6 +18,8 @@ PREFIX = 'oac_' FIELDS = ('InstallationID', *REFERENCE_FIELDS) +PENDING_PAUSE_SETTLE_SECONDS = 600 # a pause request finishes in seconds; ten minutes proves it did not take effect + # Exits 78 unless the template carries the startup script and a contract for this Runtime bootstrap version. TEMPLATE_CHECK = """import importlib.util, os, sys root, version = sys.argv[1], int(sys.argv[2]) @@ -388,13 +390,17 @@ def suspend(self): cloud = self.current(source) if cloud.state == 'running' and not request['ObserveOnly']: # Record the attempt before the native call; pause of a paused sandbox is a no-op. - self.receipt.save(pause={'operation_id': operation, 'status': 'pending'}) + self.receipt.save(pause={'operation_id': operation, 'status': 'pending', 'at': time.time()}) Sandbox.pause(cloud.sandbox_id, **self.options()) cloud = self.qualified(self.owns(Sandbox.get_info(cloud.sandbox_id, **self.options()))) pause = (self.receipt.data or {}).get('pause') or {} if cloud.state != 'paused' or pause.get('operation_id') != operation: # A dispatched pause may still take effect, so running is never reported for it. pending = pause.get('operation_id') == operation and pause.get('status') == 'pending' + if pending and request['ObserveOnly'] and cloud.state == 'running' and time.time() - pause.get('at', time.time()) > PENDING_PAUSE_SETTLE_SECONDS: + # Observation proves the long-dispatched pause did not take effect; close it as not taken. + self.receipt.save(pause={'operation_id': operation, 'status': 'not_taken'}) + return self.state(source, cloud) if request['ObserveOnly'] and not pending: return self.state(source, cloud) raise Failure('unconfirmed') diff --git a/services/core/tools/e2b-provider/provider_test.py b/services/core/tools/e2b-provider/provider_test.py index 76b27870f..b2a3e7877 100644 --- a/services/core/tools/e2b-provider/provider_test.py +++ b/services/core/tools/e2b-provider/provider_test.py @@ -425,6 +425,15 @@ def test_failure_after_dispatch_stays_open(self): self.assertEqual(self.record()['resume']['status'], 'dispatched') self.assertNotIn(self.op, self.record().get('closed', {})) + def test_old_pending_pause_is_settled_by_observation(self): + self.assertEqual(self.suspend()['ErrorCode'], 'unconfirmed') + self.assertEqual(self.suspend(observe=True)['ErrorCode'], 'unconfirmed') + with patch('provider.time.time', return_value=time.time() + provider.PENDING_PAUSE_SETTLE_SECONDS + 1): + result = self.suspend(observe=True) + self.assertEqual((result['ErrorCode'], result['State']['Status']), ('', 'running')) + self.assertNotIn('Snapshot', result['State']) + self.assertEqual(self.record()['pause']['status'], 'not_taken') + def test_renew_compute_extends_only_a_running_lease(self): self.request['Compute'] = self.compute() self.assertEqual(self.call('renew_compute')['ErrorCode'], '') From e1698284c007711eac6d045a8420f8bfe33a42dd Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sat, 24 Oct 2026 01:37:35 +0000 Subject: [PATCH 11/12] Report running sandboxes in the Web acceptance fixture --- apps/web/e2e/fixture-console.mjs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/web/e2e/fixture-console.mjs b/apps/web/e2e/fixture-console.mjs index e2fa694f3..a6c92d720 100644 --- a/apps/web/e2e/fixture-console.mjs +++ b/apps/web/e2e/fixture-console.mjs @@ -82,18 +82,18 @@ function nodeRollout(previous = 0) { } function unconfiguredDeployment(generation = 0, ownerEpoch = 3) { - return { installation_id: INSTALLATION_ID, provider: "", credential_configured: false, core_url: publicUrl(), reset: null, rollout: noNodeRollout(), owner_epoch: ownerEpoch, generation, mode: "", resources: { allocations: 0, pending: 0 }, suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "", credential_configured: false, core_url: publicUrl(), reset: null, rollout: noNodeRollout(), owner_epoch: ownerEpoch, generation, mode: "", resources: { allocations: 0, running: 0, pending: 0 }, suspension: null }; } function configuredDeployment() { - return { installation_id: INSTALLATION_ID, provider: "docker", credential_configured: false, configuration: {}, metadata: {}, core_url: publicUrl(), reset: null, rollout: nodeRollout(), owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 } }, specification_digest: "fixture", suspension: null }; + return { installation_id: INSTALLATION_ID, provider: "docker", credential_configured: false, configuration: {}, metadata: {}, core_url: publicUrl(), reset: null, rollout: nodeRollout(), owner_epoch: 3, generation: 1, mode: "nodes", resources: { allocations: 0, running: 0, pending: 0 }, specification: { resources: { cpus: 2, memory_mib: 4096 } }, specification_digest: "fixture", suspension: null }; } /** The E2B template build as Core read it when the selection was saved. */ const templateBuild = { status: "ready", resources: { cpus: 2, memory_mib: 2048, root_disk_mib: 10240 } }; // E2B runs sandboxes in its cloud: no nodes, only what Core holds there. function e2bDeployment() { - return { ...configuredDeployment(), provider: "e2b", mode: "direct", rollout: noNodeRollout(), resources: { allocations: 3, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, configuration: { template: "oac-runtime:0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b", api_url: "https://api.e2b.app", domain: "e2b.app" } , credential_configured: true, metadata: { template_build: templateBuild } }; + return { ...configuredDeployment(), provider: "e2b", mode: "direct", rollout: noNodeRollout(), resources: { allocations: 3, running: 3, pending: 1 }, specification: { resources: { cpus: 2, memory_mib: 2048 } }, configuration: { template: "oac-runtime:0f1e2d3c-4b5a-6978-8a9b-0c1d2e3f4a5b", api_url: "https://api.e2b.app", domain: "e2b.app" } , credential_configured: true, metadata: { template_build: templateBuild } }; } function reset(mode = "login", fresh = false, sandbox = "configured", nodes = "demo", address = "public", installers = true, artifacts = "docker,microsandbox") { @@ -419,7 +419,7 @@ async function sandboxRoute(request, response, path, url) { if (!initialize && !e2b) { for (const node of state.nodes) node.rollout = { state: node.online ? "preparing" : "unknown", ready_generation: node.rollout.ready_generation }; } - const held = initialize ? { allocations: 0, pending: 0 } : previous.resources; + const held = initialize ? { allocations: 0, running: 0, pending: 0 } : previous.resources; state.deployment = { ...configuredDeployment(), provider: input.provider, mode: e2b ? "direct" : "nodes", generation: previous.generation + 1, owner_epoch: previous.owner_epoch, From abe4da58963398f780e761ac7f796a8432df46a1 Mon Sep 17 00:00:00 2001 From: SaladDay <1203511142@qq.com> Date: Sat, 24 Oct 2026 02:03:06 +0000 Subject: [PATCH 12/12] Fix the second review findings on checkpoint suspension Running compute renews its native lease under one condition, an open Runtime connection, which is the existing renewal condition. Sandbox metrics counts running sandboxes without suspended ones, and the Provider and deployment docs describe Bootstrap.SuspendControl and resources.running. --- apps/web/src/features/metrics/SandboxMetricsPage.tsx | 2 +- apps/web/src/i18n/locales/en/metrics.ts | 4 ++-- apps/web/src/i18n/locales/zh-CN/metrics.ts | 4 ++-- contracts/agents-api/sandbox-deployment.md | 2 +- contracts/agents-api/zh/sandbox-deployment.md | 4 ++-- docs/sandbox-provider.md | 2 +- docs/web/console-api-usage.md | 2 +- docs/zh/sandbox-provider.md | 4 ++-- docs/zh/web/console-api-usage.md | 4 ++-- services/core/internal/execution/runtime_compute.go | 4 ++-- services/core/internal/execution/runtime_lifecycle.go | 9 ++++++++- services/core/tools/e2b-provider/provider.py | 2 +- 12 files changed, 25 insertions(+), 18 deletions(-) diff --git a/apps/web/src/features/metrics/SandboxMetricsPage.tsx b/apps/web/src/features/metrics/SandboxMetricsPage.tsx index d63f84329..b0f5160d4 100644 --- a/apps/web/src/features/metrics/SandboxMetricsPage.tsx +++ b/apps/web/src/features/metrics/SandboxMetricsPage.tsx @@ -145,7 +145,7 @@ function CloudSection({ deployment }: { deployment: SandboxDeployment }) { >

{t("sandbox.cloud.counts")}

- +