From 85c08472f2c04808fd6c681711f2ebdb954ca5dd Mon Sep 17 00:00:00 2001 From: Paulo Date: Sun, 18 Oct 2026 16:51:00 +0000 Subject: [PATCH] Fail the host on any provider error in provisioning Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01M34UnuQrWKXnCqmUXDQf58 --- src/hosts/service.py | 4 +-- src/hosts/tests/test_models.py | 31 ++++++++++++------- src/providers/cloudflare/provider.py | 19 +++++------- .../cloudflare/tests/test_provider.py | 17 +++++++--- src/providers/daytona/provider.py | 17 +++------- src/providers/e2b/provider.py | 13 ++------ src/providers/vercel/provider.py | 17 +++------- 7 files changed, 55 insertions(+), 63 deletions(-) diff --git a/src/hosts/service.py b/src/hosts/service.py index e3c7d8a..92bd3c6 100644 --- a/src/hosts/service.py +++ b/src/hosts/service.py @@ -35,10 +35,8 @@ from providers.base import VMProvider from providers.environment import get_persist from providers.exceptions import ( - ProviderCommandError, ProviderError, ProviderNotFoundError, - ProviderTransportError, UnknownProviderError, UnsupportedSizingError, ) @@ -604,7 +602,7 @@ async def provision(self, host_id: str) -> None: instance_type=host.instance_type, disk_gb=host.disk_gb, ) - except (ProviderCommandError, ProviderTransportError) as exc: + except ProviderError as exc: await self.mark_failed(host, exc) return diff --git a/src/hosts/tests/test_models.py b/src/hosts/tests/test_models.py index 02d8d6e..7db1355 100644 --- a/src/hosts/tests/test_models.py +++ b/src/hosts/tests/test_models.py @@ -1,18 +1,22 @@ import uuid +from datetime import UTC, datetime, timedelta, timezone from types import SimpleNamespace from unittest.mock import AsyncMock import pytest +from sqlalchemy import select from uuid6 import uuid7 from core.database import async_session_factory -from hosts.models import Host +from hosts.exceptions import ProvisioningFailedError +from hosts.models import Host, HostStatus from hosts.service import HostService, utc_now from networking.tailscale import ( DeviceDiscoveryTimeoutError, JoinCredentials, ) from providers.base import VMCreateResult +from providers.exceptions import ProviderAuthError from providers.exe.settings import ExeSettings @@ -195,8 +199,6 @@ async def test_create_host_clears_safety_ttl_when_caller_omits_one(monkeypatch): async def test_create_host_lands_on_caller_ttl_on_success(monkeypatch): - from datetime import timedelta - await _patch_provision_happy_path(monkeypatch) caller_ttl = utc_now() + timedelta(hours=2) @@ -211,8 +213,6 @@ async def test_create_host_expires_errored_host_for_janitor(monkeypatch): # Strand protection on the unhappy path: if provisioning errors out, # expire the row immediately so the janitor reaps it (and its VM) on the # next cycle, without operators having to chase ERROR hosts manually. - from hosts.exceptions import ProvisioningFailedError - mocks = await _patch_provision_happy_path(monkeypatch) mocks["wait_for_device"].side_effect = DeviceDiscoveryTimeoutError( "Tailscale never surfaced device" @@ -229,8 +229,6 @@ async def test_create_host_expires_errored_host_for_janitor(monkeypatch): # Row should still exist, expired so the janitor reaps it next cycle. async with async_session_factory() as session: - from sqlalchemy import select - result = await session.execute(select(Host).where(Host.status == "error")) errored = result.scalar_one() assert errored.expires_at is not None @@ -255,6 +253,21 @@ async def test_provision_generated_authkey_overrides_caller_env(monkeypatch): assert mocks["create_vm"].await_args.kwargs["env"]["TAILSCALE_AUTHKEY"] == "tskey-secret" +async def test_provision_marks_error_when_provider_rejects_credentials(monkeypatch): + host = await create_host_record(name="lb-sandbox-test", status="provisioning") + mocks = await _patch_provision_happy_path(monkeypatch) + mocks["create_vm"].side_effect = ProviderAuthError("exe.dev authentication failed") + + async with async_session_factory() as session: + await HostService(session).provision(str(host.id)) + + async with async_session_factory() as session: + refreshed = await session.get(Host, host.id) + assert refreshed is not None + assert refreshed.status == "error" + assert refreshed.last_error == "ProviderAuthError: exe.dev authentication failed" + + async def test_provision_marks_error_when_device_discovery_times_out(monkeypatch): host = await create_host_record(name="lb-sandbox-test", status="provisioning") mocks = await _patch_provision_happy_path(monkeypatch) @@ -541,10 +554,6 @@ async def create_host_record( async def test_expires_at_with_non_utc_offset_round_trips_as_utc(): # SQLite drops tz offsets, so the bind-time UTC normalization must preserve # the instant: 2099-01-01T00:00-08:00 must read back as 08:00Z, not 00:00Z. - from datetime import UTC, datetime, timedelta, timezone - - from hosts.models import HostStatus - offset_expires = datetime(2099, 1, 1, 0, 0, tzinfo=timezone(timedelta(hours=-8))) host = Host( id=uuid7(), diff --git a/src/providers/cloudflare/provider.py b/src/providers/cloudflare/provider.py index 5e6edf1..7b20f82 100644 --- a/src/providers/cloudflare/provider.py +++ b/src/providers/cloudflare/provider.py @@ -3,7 +3,7 @@ from core.settings import get_settings from providers import environment from providers.base import VMCreateResult, VMProvider -from providers.exceptions import ProviderAuthError, ProviderCommandError +from providers.exceptions import ProviderCommandError from .api import CloudflareAPI from .settings import CloudflareSettings @@ -46,16 +46,13 @@ async def create_vm( ) -> VMCreateResult: if not setup_script: raise ProviderCommandError("Cloudflare requires TAILSCALE_ENABLED=true") - try: - await self.api.create_sandbox( - name, - image=image, - instance=instance_type or self.settings.instance_type, - script=environment.get_cloud_init(setup_script, env), - label=self.service_label, - ) - except ProviderAuthError as exc: - raise ProviderCommandError(str(exc)) from exc + await self.api.create_sandbox( + name, + image=image, + instance=instance_type or self.settings.instance_type, + script=environment.get_cloud_init(setup_script, env), + label=self.service_label, + ) return VMCreateResult(provider_id=name, name=name, ssh_username="root") async def delete_vm(self, name: str) -> None: diff --git a/src/providers/cloudflare/tests/test_provider.py b/src/providers/cloudflare/tests/test_provider.py index bc38267..205358d 100644 --- a/src/providers/cloudflare/tests/test_provider.py +++ b/src/providers/cloudflare/tests/test_provider.py @@ -8,7 +8,12 @@ from providers.cloudflare.api import CloudflareAPI from providers.cloudflare.provider import CloudflareProvider from providers.cloudflare.settings import CloudflareSettings -from providers.exceptions import ProviderCommandError, ProviderTransportError +from providers.exceptions import ( + ProviderAuthError, + ProviderCommandError, + ProviderError, + ProviderTransportError, +) from providers.registry import get_provider_names @@ -65,10 +70,14 @@ async def test_worker_failure_is_a_transport_error_without_its_body( @respx.mock -@pytest.mark.parametrize("status", [400, 401]) -async def test_rejected_request_is_a_command_error(provider: CloudflareProvider, status: int): +@pytest.mark.parametrize( + ("status", "error"), [(400, ProviderCommandError), (401, ProviderAuthError)] +) +async def test_rejected_request_is_a_typed_error( + provider: CloudflareProvider, status: int, error: type[ProviderError] +): respx.put("https://worker.test/sandboxes/sb-test").respond(status) - with pytest.raises(ProviderCommandError): + with pytest.raises(error): await provider.create_vm(name="sb-test", image="base", setup_script="true") diff --git a/src/providers/daytona/provider.py b/src/providers/daytona/provider.py index b1f32b8..07b15fa 100644 --- a/src/providers/daytona/provider.py +++ b/src/providers/daytona/provider.py @@ -3,11 +3,7 @@ from core.settings import get_settings from providers import environment from providers.base import VMCreateResult, VMProvider -from providers.exceptions import ( - ProviderAuthError, - ProviderCommandError, - ProviderNotFoundError, -) +from providers.exceptions import ProviderCommandError from .api import DaytonaAPI from .settings import DaytonaSettings @@ -47,13 +43,10 @@ async def create_vm( ) -> VMCreateResult: if not setup_script: raise ProviderCommandError("Daytona requires TAILSCALE_ENABLED=true") - try: - await self.api.check_snapshot(image) - await self.api.create_sandbox(name, image=image, label=self.service_label) - sandbox = await self.api.wait_for_started(name) - await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env)) - except (ProviderAuthError, ProviderNotFoundError) as exc: - raise ProviderCommandError(str(exc)) from exc + await self.api.check_snapshot(image) + await self.api.create_sandbox(name, image=image, label=self.service_label) + sandbox = await self.api.wait_for_started(name) + await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env)) return VMCreateResult(provider_id=sandbox["id"], name=name, ssh_username="root") async def delete_vm(self, name: str) -> None: diff --git a/src/providers/e2b/provider.py b/src/providers/e2b/provider.py index 9f6f28d..32e0366 100644 --- a/src/providers/e2b/provider.py +++ b/src/providers/e2b/provider.py @@ -4,11 +4,7 @@ from core.settings import get_settings from providers import environment from providers.base import VMCreateResult, VMProvider -from providers.exceptions import ( - ProviderAuthError, - ProviderCommandError, - ProviderNotFoundError, -) +from providers.exceptions import ProviderCommandError from .api import E2BAPI from .settings import E2BSettings @@ -49,11 +45,8 @@ async def create_vm( ) -> VMCreateResult: if not setup_script: raise ProviderCommandError("E2B requires TAILSCALE_ENABLED=true") - try: - sandbox = await self.api.create_sandbox(name, image=image, label=self.service_label) - await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env)) - except (ProviderAuthError, ProviderNotFoundError) as exc: - raise ProviderCommandError(str(exc)) from exc + sandbox = await self.api.create_sandbox(name, image=image, label=self.service_label) + await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env)) return VMCreateResult(provider_id=sandbox.sandbox_id, name=name, ssh_username="root") async def delete_vm(self, name: str) -> None: diff --git a/src/providers/vercel/provider.py b/src/providers/vercel/provider.py index d08943d..a66acc3 100644 --- a/src/providers/vercel/provider.py +++ b/src/providers/vercel/provider.py @@ -4,11 +4,7 @@ from core.settings import get_settings from providers import environment from providers.base import VMCreateResult, VMProvider -from providers.exceptions import ( - ProviderAuthError, - ProviderCommandError, - ProviderNotFoundError, -) +from providers.exceptions import ProviderCommandError from .api import VercelAPI from .settings import VercelSettings @@ -54,13 +50,10 @@ async def create_vm( vcpus = int(instance_type or self.settings.vcpus) except ValueError: raise ProviderCommandError("Vercel instance_type must be a vCPU count") from None - try: - session_id = await self.api.create_sandbox( - name, image=image, vcpus=vcpus, label=self.service_label - ) - await self.api.bootstrap(session_id, environment.get_cloud_init(setup_script, env)) - except (ProviderAuthError, ProviderNotFoundError) as exc: - raise ProviderCommandError(str(exc)) from exc + session_id = await self.api.create_sandbox( + name, image=image, vcpus=vcpus, label=self.service_label + ) + await self.api.bootstrap(session_id, environment.get_cloud_init(setup_script, env)) return VMCreateResult(provider_id=session_id, name=name, ssh_username="root") async def delete_vm(self, name: str) -> None: