Fail the host on any provider error in provisioning

src/hosts/service.py

@@ -35,10 +35,8 @@
 from providers.base import VMProvider
 from providers.environment import get_persist
 from providers.exceptions import (
-    ProviderCommandError,
     ProviderError,
     ProviderNotFoundError,
-    ProviderTransportError,
     UnknownProviderError,
     UnsupportedSizingError,
 )
@@ -604,7 +602,7 @@ async def provision(self, host_id: str) -> None:
                 instance_type=host.instance_type,
                 disk_gb=host.disk_gb,
             )
-        except (ProviderCommandError, ProviderTransportError) as exc:
+        except ProviderError as exc:
             await self.mark_failed(host, exc)
             return
 

src/hosts/tests/test_models.py

@@ -1,18 +1,22 @@
 import uuid
+from datetime import UTC, datetime, timedelta, timezone
 from types import SimpleNamespace
 from unittest.mock import AsyncMock
 
 import pytest
+from sqlalchemy import select
 from uuid6 import uuid7
 
 from core.database import async_session_factory
-from hosts.models import Host
+from hosts.exceptions import ProvisioningFailedError
+from hosts.models import Host, HostStatus
 from hosts.service import HostService, utc_now
 from networking.tailscale import (
     DeviceDiscoveryTimeoutError,
     JoinCredentials,
 )
 from providers.base import VMCreateResult
+from providers.exceptions import ProviderAuthError
 from providers.exe.settings import ExeSettings
 
 
@@ -195,8 +199,6 @@ async def test_create_host_clears_safety_ttl_when_caller_omits_one(monkeypatch):
 
 
 async def test_create_host_lands_on_caller_ttl_on_success(monkeypatch):
-    from datetime import timedelta
-
     await _patch_provision_happy_path(monkeypatch)
     caller_ttl = utc_now() + timedelta(hours=2)
 
@@ -211,8 +213,6 @@ async def test_create_host_expires_errored_host_for_janitor(monkeypatch):
     # Strand protection on the unhappy path: if provisioning errors out,
     # expire the row immediately so the janitor reaps it (and its VM) on the
     # next cycle, without operators having to chase ERROR hosts manually.
-    from hosts.exceptions import ProvisioningFailedError
-
     mocks = await _patch_provision_happy_path(monkeypatch)
     mocks["wait_for_device"].side_effect = DeviceDiscoveryTimeoutError(
         "Tailscale never surfaced device"
@@ -229,8 +229,6 @@ async def test_create_host_expires_errored_host_for_janitor(monkeypatch):
 
     # Row should still exist, expired so the janitor reaps it next cycle.
     async with async_session_factory() as session:
-        from sqlalchemy import select
-
         result = await session.execute(select(Host).where(Host.status == "error"))
         errored = result.scalar_one()
         assert errored.expires_at is not None
@@ -255,6 +253,21 @@ async def test_provision_generated_authkey_overrides_caller_env(monkeypatch):
     assert mocks["create_vm"].await_args.kwargs["env"]["TAILSCALE_AUTHKEY"] == "tskey-secret"
 
 
+async def test_provision_marks_error_when_provider_rejects_credentials(monkeypatch):
+    host = await create_host_record(name="lb-sandbox-test", status="provisioning")
+    mocks = await _patch_provision_happy_path(monkeypatch)
+    mocks["create_vm"].side_effect = ProviderAuthError("exe.dev authentication failed")
+
+    async with async_session_factory() as session:
+        await HostService(session).provision(str(host.id))
+
+    async with async_session_factory() as session:
+        refreshed = await session.get(Host, host.id)
+        assert refreshed is not None
+        assert refreshed.status == "error"
+        assert refreshed.last_error == "ProviderAuthError: exe.dev authentication failed"
+
+
 async def test_provision_marks_error_when_device_discovery_times_out(monkeypatch):
     host = await create_host_record(name="lb-sandbox-test", status="provisioning")
     mocks = await _patch_provision_happy_path(monkeypatch)
@@ -541,10 +554,6 @@ async def create_host_record(
 async def test_expires_at_with_non_utc_offset_round_trips_as_utc():
     # SQLite drops tz offsets, so the bind-time UTC normalization must preserve
     # the instant: 2099-01-01T00:00-08:00 must read back as 08:00Z, not 00:00Z.
-    from datetime import UTC, datetime, timedelta, timezone
-
-    from hosts.models import HostStatus
-
     offset_expires = datetime(2099, 1, 1, 0, 0, tzinfo=timezone(timedelta(hours=-8)))
     host = Host(
         id=uuid7(),

src/providers/cloudflare/provider.py

@@ -3,7 +3,7 @@
 from core.settings import get_settings
 from providers import environment
 from providers.base import VMCreateResult, VMProvider
-from providers.exceptions import ProviderAuthError, ProviderCommandError
+from providers.exceptions import ProviderCommandError
 
 from .api import CloudflareAPI
 from .settings import CloudflareSettings
@@ -46,16 +46,13 @@ async def create_vm(
     ) -> VMCreateResult:
         if not setup_script:
             raise ProviderCommandError("Cloudflare requires TAILSCALE_ENABLED=true")
-        try:
-            await self.api.create_sandbox(
-                name,
-                image=image,
-                instance=instance_type or self.settings.instance_type,
-                script=environment.get_cloud_init(setup_script, env),
-                label=self.service_label,
-            )
-        except ProviderAuthError as exc:
-            raise ProviderCommandError(str(exc)) from exc
+        await self.api.create_sandbox(
+            name,
+            image=image,
+            instance=instance_type or self.settings.instance_type,
+            script=environment.get_cloud_init(setup_script, env),
+            label=self.service_label,
+        )
         return VMCreateResult(provider_id=name, name=name, ssh_username="root")
 
     async def delete_vm(self, name: str) -> None:

src/providers/cloudflare/tests/test_provider.py

@@ -8,7 +8,12 @@
 from providers.cloudflare.api import CloudflareAPI
 from providers.cloudflare.provider import CloudflareProvider
 from providers.cloudflare.settings import CloudflareSettings
-from providers.exceptions import ProviderCommandError, ProviderTransportError
+from providers.exceptions import (
+    ProviderAuthError,
+    ProviderCommandError,
+    ProviderError,
+    ProviderTransportError,
+)
 from providers.registry import get_provider_names
 
 
@@ -65,10 +70,14 @@ async def test_worker_failure_is_a_transport_error_without_its_body(
 
 
 @respx.mock
-@pytest.mark.parametrize("status", [400, 401])
-async def test_rejected_request_is_a_command_error(provider: CloudflareProvider, status: int):
+@pytest.mark.parametrize(
+    ("status", "error"), [(400, ProviderCommandError), (401, ProviderAuthError)]
+)
+async def test_rejected_request_is_a_typed_error(
+    provider: CloudflareProvider, status: int, error: type[ProviderError]
+):
     respx.put("https://worker.test/sandboxes/sb-test").respond(status)
-    with pytest.raises(ProviderCommandError):
+    with pytest.raises(error):
         await provider.create_vm(name="sb-test", image="base", setup_script="true")
 
 

src/providers/daytona/provider.py

@@ -3,11 +3,7 @@
 from core.settings import get_settings
 from providers import environment
 from providers.base import VMCreateResult, VMProvider
-from providers.exceptions import (
-    ProviderAuthError,
-    ProviderCommandError,
-    ProviderNotFoundError,
-)
+from providers.exceptions import ProviderCommandError
 
 from .api import DaytonaAPI
 from .settings import DaytonaSettings
@@ -47,13 +43,10 @@ async def create_vm(
     ) -> VMCreateResult:
         if not setup_script:
             raise ProviderCommandError("Daytona requires TAILSCALE_ENABLED=true")
-        try:
-            await self.api.check_snapshot(image)
-            await self.api.create_sandbox(name, image=image, label=self.service_label)
-            sandbox = await self.api.wait_for_started(name)
-            await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env))
-        except (ProviderAuthError, ProviderNotFoundError) as exc:
-            raise ProviderCommandError(str(exc)) from exc
+        await self.api.check_snapshot(image)
+        await self.api.create_sandbox(name, image=image, label=self.service_label)
+        sandbox = await self.api.wait_for_started(name)
+        await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env))
         return VMCreateResult(provider_id=sandbox["id"], name=name, ssh_username="root")
 
     async def delete_vm(self, name: str) -> None:

src/providers/e2b/provider.py

@@ -4,11 +4,7 @@
 from core.settings import get_settings
 from providers import environment
 from providers.base import VMCreateResult, VMProvider
-from providers.exceptions import (
-    ProviderAuthError,
-    ProviderCommandError,
-    ProviderNotFoundError,
-)
+from providers.exceptions import ProviderCommandError
 
 from .api import E2BAPI
 from .settings import E2BSettings
@@ -49,11 +45,8 @@ async def create_vm(
     ) -> VMCreateResult:
         if not setup_script:
             raise ProviderCommandError("E2B requires TAILSCALE_ENABLED=true")
-        try:
-            sandbox = await self.api.create_sandbox(name, image=image, label=self.service_label)
-            await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env))
-        except (ProviderAuthError, ProviderNotFoundError) as exc:
-            raise ProviderCommandError(str(exc)) from exc
+        sandbox = await self.api.create_sandbox(name, image=image, label=self.service_label)
+        await self.api.bootstrap(sandbox, environment.get_cloud_init(setup_script, env))
         return VMCreateResult(provider_id=sandbox.sandbox_id, name=name, ssh_username="root")
 
     async def delete_vm(self, name: str) -> None:

src/providers/vercel/provider.py

@@ -4,11 +4,7 @@
 from core.settings import get_settings
 from providers import environment
 from providers.base import VMCreateResult, VMProvider
-from providers.exceptions import (
-    ProviderAuthError,
-    ProviderCommandError,
-    ProviderNotFoundError,
-)
+from providers.exceptions import ProviderCommandError
 
 from .api import VercelAPI
 from .settings import VercelSettings
@@ -54,13 +50,10 @@ async def create_vm(
             vcpus = int(instance_type or self.settings.vcpus)
         except ValueError:
             raise ProviderCommandError("Vercel instance_type must be a vCPU count") from None
-        try:
-            session_id = await self.api.create_sandbox(
-                name, image=image, vcpus=vcpus, label=self.service_label
-            )
-            await self.api.bootstrap(session_id, environment.get_cloud_init(setup_script, env))
-        except (ProviderAuthError, ProviderNotFoundError) as exc:
-            raise ProviderCommandError(str(exc)) from exc
+        session_id = await self.api.create_sandbox(
+            name, image=image, vcpus=vcpus, label=self.service_label
+        )
+        await self.api.bootstrap(session_id, environment.get_cloud_init(setup_script, env))
         return VMCreateResult(provider_id=session_id, name=name, ssh_username="root")
 
     async def delete_vm(self, name: str) -> None: