From b080ad43b66af111a6f708dcb8eaa241224d4da1 Mon Sep 17 00:00:00 2001 From: Pandelis Zembashis Date: Sun, 18 Oct 2026 12:40:40 -0700 Subject: [PATCH] Report startup failures before the connection wait expires --- README.md | 2 ++ builder.mjs | 7 +++++-- builder.test.mjs | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 823ab3c..9520a30 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ An organization admin must provision the builder first with `com docker-builder Inputs are `token` and `org` (required), `builder` (default `main`), `wake-timeout` (default `900` seconds, range `30`–`3600`), `use` (default `true`), `api-url` (default `https://composal.ai`), and `install-cli` (default `true`). The installed CLI is added to the job PATH. Set `install-cli: 'false'` to use an already installed released `com` supporting background Docker builder setup. Docker and Buildx must be installed on the runner. Outputs are `builder`, `context`, and `driver` (`docker`). Each job isolates CLI supervisor state in its temporary directory. Setup logs the CLI version, checks gateway health and waits for the shared builder to become ready before opening native Docker connections. Cold starts can take several minutes; `wake-timeout` controls this wait. Progress includes the builder state and generation. Setup requests a retry when it finds a builder already starting, so a failed earlier wake can resume. Canceling a job disconnects only that job; it does not cancel the shared wake. It retries transient status-read server errors, then checks the remote daemon with bounded, retryable probes and verifies Buildx readiness before selecting the context and builder for subsequent job steps using `DOCKER_CONTEXT` and `BUILDX_BUILDER`. It leaves Docker's saved defaults unchanged and restores the previous job environment during cleanup. +If a startup attempt fails, setup reports its generation, stage and failure reason immediately. A fresh job can retry a failed startup against the same builder and retained cache. Simultaneous jobs share that startup; canceling one job keeps the shared startup running. + Setup also disables Buildx metadata provenance and Docker build-record uploads/summaries: the scoped gateway does not expose BuildKit content-record downloads. Image digest metadata is retained. Previous values are restored during cleanup. In explicit selection mode, set `BUILDX_METADATA_PROVENANCE=disabled`, `DOCKER_BUILD_RECORD_UPLOAD=false` and `DOCKER_BUILD_SUMMARY=false` on the build step yourself. The normal setup works with `docker/build-push-action`, `docker buildx build`, and native Docker commands. For explicit selection only, set `use: 'false'`, then set `DOCKER_CONTEXT` to the context output and pass the builder output on every build step. The embedded Docker driver refuses a builder belonging to a different active context. Do not create a separate `docker-container` builder, add GHA/registry cache exporters, or run cache-dance: this embedded Docker driver uses its retained native cache. The current classic image store does not support those external exporters or attestations; the example disables provenance. Explicit local exports still transfer image data to CI. diff --git a/builder.mjs b/builder.mjs index db14600..8d039e5 100644 --- a/builder.mjs +++ b/builder.mjs @@ -125,15 +125,18 @@ export async function setup(env = process.env, run = command, wait = delay) { let current = resource; let requestedWake = false; while (current.status !== 'ready') { + if (current.status === 'failed' && requestedWake) { + throw new Error(`Remote builder startup failed (generation=${current.generation ?? 'unknown'}, stage=${current.startup_stage || 'unknown'}): ${current.startup_failure || 'startup did not complete; retry the job to resume the same cache'}`); + } if (Date.now() >= deadline) throw new Error(`Remote builder wake timed out after ${wakeTimeout}s (state=${current.status}, generation=${current.generation ?? 'unknown'}); the shared wake continues independently of this job`); - if (current.status === 'sleeping' || (current.status === 'starting' && !requestedWake)) { + if (current.status === 'sleeping' || (['starting', 'failed'].includes(current.status) && !requestedWake)) { await run(cli, ['docker-builder', 'wake', resource.id, ...options], cliEnv, Math.max(1, deadline - Date.now())); requestedWake = true; } else if (!['starting', 'stopping'].includes(current.status)) { throw new Error('Composal CLI returned an unexpected builder state'); } if (Date.now() >= nextProgress) { - stage(`Waiting for builder: state=${current.status}, generation=${current.generation ?? 'unknown'}`); + stage(`Waiting for builder: state=${current.status}, generation=${current.generation ?? 'unknown'}, stage=${current.startup_stage || 'unknown'}`); nextProgress = Date.now() + 15_000; } await wait(Math.min(1_000, Math.max(1, deadline - Date.now()))); diff --git a/builder.test.mjs b/builder.test.mjs index 460a61d..c799d8e 100644 --- a/builder.test.mjs +++ b/builder.test.mjs @@ -122,6 +122,38 @@ test('an unreachable gateway does not wake compute or configure Docker', async t assert.ok(!f.calls.some(c => ['wake', 'setup'].includes(c.args[1]))); }); +test('a failed startup stops polling immediately and reports its generation and stage', async t => { + const f = await fixture(t); + let statuses = 0; + const run = async (exe, args, env, timeout) => { + const result = await f.run(exe, args, env, timeout); + if (args[1] === 'status') return JSON.stringify({ docker_builder: { + id: 'docker_builder_456', organization_id: 'org_123', generation: 39, + status: ++statuses === 1 ? 'starting' : 'failed', startup_stage: 'runtime_create', + startup_failure: 'allocation timed out', gateway_url: 'wss://builder.example/v1/organizations/org_123/docker', + } }); + return result; + }; + await assert.rejects(setup(f.env, run, async () => {}), /startup failed.*generation=39.*runtime_create.*allocation timed out/); + assert.equal(statuses, 2); + assert.ok(!f.calls.some(c => c.args[1] === 'setup')); +}); + +test('a fresh job can explicitly retry a failed startup once using the same builder', async t => { + const f = await fixture(t); + let statuses = 0; + const run = async (exe, args, env, timeout) => { + const result = await f.run(exe, args, env, timeout); + if (args[1] === 'status') return JSON.stringify({ docker_builder: { + id: 'docker_builder_456', organization_id: 'org_123', generation: 39, + status: ++statuses === 1 ? 'failed' : 'ready', gateway_url: 'wss://builder.example/v1/organizations/org_123/docker', + } }); + return result; + }; + await setup(f.env, run, async () => {}); + assert.equal(f.calls.filter(c => c.args[1] === 'wake').length, 1); +}); + test('transient status failures are retried before configuring the native proxy', async t => { const f = await fixture(t); const waits = [];