From 96d9c88dc8ce07e32e49729cbb3fe7e4abea075c Mon Sep 17 00:00:00 2001 From: Pandelis Zembashis Date: Sun, 18 Oct 2026 11:15:00 -0700 Subject: [PATCH] Retry an existing starting Docker builder when connecting --- README.md | 4 ++-- builder.mjs | 4 +++- builder.test.mjs | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8cad748..823ab3c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Authenticate to your registry before selecting the remote context: the gateway d An organization admin must provision the builder first with `com docker-builder create ci --org acme`. The token needs membership in that organization. Setup fails if the builder is missing, unavailable, or refused; it never falls back to a local build or creates a cache accidentally. -Inputs are `token` and `org` (required), `builder` (default `main`), `wake-timeout` (default `900` seconds, range `30`–`3600`), `use` (default `true`), `api-url` (default `https://composal.ai`), and `install-cli` (default `true`). The installed CLI is added to the job PATH. Set `install-cli: 'false'` to use an already installed released `com` supporting background Docker builder setup. Docker and Buildx must be installed on the runner. Outputs are `builder`, `context`, and `driver` (`docker`). Each job isolates CLI supervisor state in its temporary directory. Setup logs the CLI version, checks gateway health and waits for the shared builder to become ready before opening native Docker connections. Cold starts can take several minutes; `wake-timeout` controls this wait. Progress includes the builder state and generation. Canceling a job disconnects only that job; it does not cancel the shared wake. It retries transient status-read server errors, then checks the remote daemon with bounded, retryable probes and verifies Buildx readiness before selecting the context and builder for subsequent job steps using `DOCKER_CONTEXT` and `BUILDX_BUILDER`. It leaves Docker's saved defaults unchanged and restores the previous job environment during cleanup. +Inputs are `token` and `org` (required), `builder` (default `main`), `wake-timeout` (default `900` seconds, range `30`–`3600`), `use` (default `true`), `api-url` (default `https://composal.ai`), and `install-cli` (default `true`). The installed CLI is added to the job PATH. Set `install-cli: 'false'` to use an already installed released `com` supporting background Docker builder setup. Docker and Buildx must be installed on the runner. Outputs are `builder`, `context`, and `driver` (`docker`). Each job isolates CLI supervisor state in its temporary directory. Setup logs the CLI version, checks gateway health and waits for the shared builder to become ready before opening native Docker connections. Cold starts can take several minutes; `wake-timeout` controls this wait. Progress includes the builder state and generation. Setup requests a retry when it finds a builder already starting, so a failed earlier wake can resume. Canceling a job disconnects only that job; it does not cancel the shared wake. It retries transient status-read server errors, then checks the remote daemon with bounded, retryable probes and verifies Buildx readiness before selecting the context and builder for subsequent job steps using `DOCKER_CONTEXT` and `BUILDX_BUILDER`. It leaves Docker's saved defaults unchanged and restores the previous job environment during cleanup. Setup also disables Buildx metadata provenance and Docker build-record uploads/summaries: the scoped gateway does not expose BuildKit content-record downloads. Image digest metadata is retained. Previous values are restored during cleanup. In explicit selection mode, set `BUILDX_METADATA_PROVENANCE=disabled`, `DOCKER_BUILD_RECORD_UPLOAD=false` and `DOCKER_BUILD_SUMMARY=false` on the build step yourself. @@ -42,7 +42,7 @@ The normal setup works with `docker/build-push-action`, `docker buildx build`, a Use one connection per job. Post-job cleanup disconnects the local proxy and removes its context, including after build failure. It keeps the remote builder and cache; organization idle policy sleeps compute. Runner termination can prevent cleanup. Disconnecting does not prove a remote operation has stopped: uncertain operations retain their slot until the service reconciles them or an administrator recovers the builder. -One builder can serve multiple native builds on the same daemon and retained cache. Organization concurrency defaults to one across all its builders; an admin can raise it with `com docker-builder settings --org acme --max-concurrency 4`. New VMs receive 2 CPUs and 4 GiB RAM per configured concurrent operation, with a 128 GiB cache disk. Changing the policy does not resize an already running VM: drain its builds, sleep it to checkpoint the cache, then wake it. Qualify image size, cold build memory and warm performance before switching production builds. +One builder can serve multiple native builds on the same daemon and retained cache. Organization concurrency defaults to one across all its builders; an admin can raise it with `com docker-builder settings --org acme --max-concurrency 4`. New VMs receive 2 CPUs and 4 GiB RAM per configured concurrent operation, with a 256 GiB cache disk. Changing the policy does not resize an already running VM: drain its builds, sleep it to checkpoint the cache, then wake it. Qualify image size, cold build memory and warm performance before switching production builds. Coordinate CI jobs to fit the organization's configured capacity. The single concurrency group in the example serializes jobs; multiple groups can admit more jobs when capacity has been increased. GitHub concurrency only coordinates workflows in the same repository, and concurrent work outside those groups can still be refused. Use `queue: max` to retain pending builds (up to 100); without it GitHub replaces the previous pending build. Use separate builders when workloads need independent caches. Remote builders are in preview. diff --git a/builder.mjs b/builder.mjs index 96062a0..db14600 100644 --- a/builder.mjs +++ b/builder.mjs @@ -123,10 +123,12 @@ export async function setup(env = process.env, run = command, wait = delay) { const deadline = Date.now() + wakeTimeout * 1_000; let nextProgress = Date.now() + 15_000; let current = resource; + let requestedWake = false; while (current.status !== 'ready') { if (Date.now() >= deadline) throw new Error(`Remote builder wake timed out after ${wakeTimeout}s (state=${current.status}, generation=${current.generation ?? 'unknown'}); the shared wake continues independently of this job`); - if (current.status === 'sleeping') { + if (current.status === 'sleeping' || (current.status === 'starting' && !requestedWake)) { await run(cli, ['docker-builder', 'wake', resource.id, ...options], cliEnv, Math.max(1, deadline - Date.now())); + requestedWake = true; } else if (!['starting', 'stopping'].includes(current.status)) { throw new Error('Composal CLI returned an unexpected builder state'); } diff --git a/builder.test.mjs b/builder.test.mjs index aacc7fe..460a61d 100644 --- a/builder.test.mjs +++ b/builder.test.mjs @@ -89,6 +89,24 @@ test('a cold builder reaches ready before any native Docker connection is config assert.equal(f.calls[health].env.COMPOSAL_API_TOKEN, undefined); }); +test('an already starting builder requests one wake retry instead of only polling abandoned state', async t => { + const f = await fixture(t); + let statuses = 0; + const run = async (exe, args, env, timeout) => { + const result = await f.run(exe, args, env, timeout); + if (args[1] === 'status') return JSON.stringify({ docker_builder: { + id: 'docker_builder_456', organization_id: 'org_123', generation: 37, + status: ++statuses < 4 ? 'starting' : 'ready', + gateway_url: 'wss://builder.example/v1/organizations/org_123/docker', + } }); + if (args[1] === 'setup') assert.equal(statuses, 4); + return result; + }; + await setup(f.env, run, async () => {}); + assert.equal(f.calls.filter(c => c.args[1] === 'wake').length, 1); + assert.equal(f.calls.find(c => c.args[1] === 'wake').args[2], 'docker_builder_456'); +}); + test('an unreachable gateway does not wake compute or configure Docker', async t => { const f = await fixture(t); const run = async (exe, args, env, timeout) => {