CI: the server is deployed as its own user, sandboxed, on loopback

.github/workflows/ci.yml

@@ -345,16 +345,19 @@ jobs:
         run: cabal test nova-cache-bzip2-test --enable-tests
 
   # ---------------------------------------------------------------------------
-  # Deploy (main branch only): build on the hosted arm64 image, ship the
-  # binary.  anv26 is an Ampere box on the same Ubuntu 24.04, so the
-  # runner's glibc matches - and the production host deliberately carries
-  # no compiler toolchain.
+  # Server (arm64): build the server on the hosted arm64 image, start it
+  # there under deploy/nova-cache.service, and on a push to main ship both
+  # to the public cache's host.  anv26 is an Ampere box on the same Ubuntu
+  # 24.04, so the runner's glibc matches and its systemd is the release the
+  # host runs, and the production host deliberately carries no compiler
+  # toolchain.  The smoke test runs on every pull request, so a unit the
+  # server cannot run under fails before it merges, and a deploy installs
+  # only what just ran.
   # ---------------------------------------------------------------------------
 
-  deploy-server:
-    name: Deploy Server
+  server:
+    name: Server (arm64)
     needs: [build]
-    if: github.ref == 'refs/heads/main' && github.event_name == 'push'
     runs-on: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@v7
@@ -377,7 +380,11 @@ jobs:
           cabal build --flag server --ghc-options="-Wall -Werror"
           cp "$(cabal list-bin --flag server nova-cache-server)" /tmp/nova-cache-server
 
+      - name: Run the server under its unit
+        run: deploy/smoke-test.sh /tmp/nova-cache-server
+
       - name: Deploy to OCI
+        if: github.ref == 'refs/heads/main' && github.event_name == 'push'
         env:
           OCI_SSH_KEY: ${{ secrets.OCI_SSH_KEY }}
           CACHE_API_KEY: ${{ secrets.NOVA_CACHE_API_KEY }}
@@ -407,47 +414,34 @@ jobs:
             StrictHostKeyChecking accept-new
           SSHCFG
 
-          # Render the service unit.  The API key is not embedded in the
-          # unit: systemd loads it from /etc/nix-cache/env (written below,
-          # root-only), so the unit file carries no credentials.
-          cat > /tmp/nova-cache.service <<'EOF'
-          [Unit]
-          Description=Nova Cache Server
-          After=network.target
-
-          [Service]
-          Type=simple
-          User=ubuntu
-          Group=ubuntu
-          ExecStart=/usr/local/bin/nova-cache-server --port 5000 --store /var/lib/nix-cache
-          Restart=always
-          RestartSec=5
-          Environment=PORT=5000
-          Environment=NIX_CACHE_DIR=/var/lib/nix-cache
-          Environment=SIGNING_KEY_FILE=/etc/nix-cache/secret-key
-          EnvironmentFile=/etc/nix-cache/env
-
-          [Install]
-          WantedBy=multi-user.target
-          EOF
-
-          scp /tmp/nova-cache.service anv26:/tmp/nova-cache.service
+          # The unit, the user it runs as and Caddy's restart policy are the
+          # files of deploy/, the first two as the smoke test above ran them.
           # The binary rides the same Access tunnel as everything else; the
           # service keeps serving during the transfer and stops only for
           # the install below.
-          scp /tmp/nova-cache-server anv26:/tmp/nova-cache-server
+          scp deploy/nova-cache.service deploy/nova-cache.sysusers deploy/caddy-restart.conf /tmp/nova-cache-server anv26:/tmp/
 
           # Provision the API key as a root-only environment file.  The value
           # travels over stdin, so it appears in no command line, no log, and
           # no file on the runner.
           printf 'CACHE_API_KEY=%s\n' "$CACHE_API_KEY" | ssh anv26 \
             "sudo sh -c 'umask 077; mkdir -p /etc/nix-cache; cat > /etc/nix-cache/env; chown root:root /etc/nix-cache/env; chmod 600 /etc/nix-cache/env'"
 
+          # The signing key becomes root's: the unit hands the service a copy
+          # as a credential, so the service's user does not read the file.
+          # The store changes hands when the unit starts.  The last command
+          # fails the deploy if the server does not answer.
           ssh anv26 "\
+            sudo install -D -m 0644 /tmp/nova-cache.sysusers /etc/sysusers.d/nova-cache.conf && \
+            sudo systemd-sysusers /etc/sysusers.d/nova-cache.conf && \
+            sudo chown root:root /etc/nix-cache/secret-key && \
+            sudo chmod 600 /etc/nix-cache/secret-key && \
+            sudo install -D -m 0644 /tmp/caddy-restart.conf /etc/systemd/system/caddy.service.d/restart.conf && \
             sudo systemctl stop nova-cache.service && \
             sudo install -m 0755 /tmp/nova-cache-server /usr/local/bin/nova-cache-server && \
-            sudo cp /tmp/nova-cache.service /etc/systemd/system/nova-cache.service && \
-            rm -f /tmp/nova-cache-server /tmp/nova-cache.service && \
+            sudo install -m 0644 /tmp/nova-cache.service /etc/systemd/system/nova-cache.service && \
+            rm -f /tmp/nova-cache-server /tmp/nova-cache.service /tmp/nova-cache.sysusers /tmp/caddy-restart.conf && \
             sudo systemctl daemon-reload && \
             sudo systemctl start nova-cache.service && \
-            sudo systemctl status nova-cache.service --no-pager"
+            sudo systemctl status nova-cache.service --no-pager && \
+            curl --fail --silent --show-error --output /dev/null --retry 10 --retry-connrefused --retry-delay 1 http://127.0.0.1:5000/nix-cache-info"

deploy/caddy-restart.conf

@@ -0,0 +1,5 @@
+# Caddy's packaged unit names no restart policy, so a Caddy that exits stays
+# down, and the public cache with it.
+[Service]
+Restart=on-failure
+RestartSec=5

deploy/nova-cache.service

@@ -0,0 +1,61 @@
+# nova-cache-server on the public cache's host: its own user, one writable
+# directory, and loopback alone.  The deploy job installs this file after
+# deploy/smoke-test.sh has started the server under it.
+
+[Unit]
+Description=Nova Cache Server
+After=network.target
+
+[Service]
+Type=simple
+User=nova-cache
+Group=nova-cache
+ExecStart=/usr/local/bin/nova-cache-server --host 127.0.0.1 --port 5000 --store %S/nix-cache
+Restart=always
+RestartSec=5
+
+# /var/lib/nix-cache.  systemd creates it if it is absent and gives it, with
+# everything under it, to the service's user, whoever owned it before.
+StateDirectory=nix-cache
+
+# The signing key reaches the service as a credential: systemd reads the file
+# as root and shows a copy to this unit alone, so the file stays root-only.
+# The write key comes from an environment file only root reads.
+LoadCredential=signing-key:/etc/nix-cache/secret-key
+Environment=SIGNING_KEY_FILE=%d/signing-key
+EnvironmentFile=/etc/nix-cache/env
+
+# The reverse proxy on the same host is the service's only client.
+IPAddressDeny=any
+IPAddressAllow=localhost
+RestrictAddressFamilies=AF_INET AF_INET6
+
+# The service reads its binary and its two keys, writes its state directory
+# and listens on one port.
+NoNewPrivileges=yes
+ProtectSystem=strict
+ProtectHome=yes
+PrivateTmp=yes
+PrivateDevices=yes
+PrivateUsers=yes
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+ProtectProc=invisible
+ProcSubset=pid
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+RemoveIPC=yes
+CapabilityBoundingSet=
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallFilter=~@privileged @resources
+
+[Install]
+WantedBy=multi-user.target

deploy/nova-cache.sysusers

@@ -0,0 +1,2 @@
+# The user nova-cache.service runs as.
+u nova-cache - "nova-cache binary cache server" /var/lib/nix-cache /usr/sbin/nologin

deploy/smoke-test.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# Start nova-cache.service on this machine as the deploy installs it, and
+# check what the unit's sandbox could break: that the server starts, reads
+# its signing key, accepts an authenticated write and refuses another, that
+# a store another user owned becomes the service's, and that the server
+# answers on loopback alone.
+#
+# The keys are made here and stand in for the host's.  The machine is a
+# throwaway: the script installs into /usr/local/bin, /etc and /var/lib and
+# removes nothing.
+#
+# Usage: smoke-test.sh SERVER
+#   SERVER   the nova-cache-server binary to install
+set -euo pipefail
+
+server=${1:?the nova-cache-server binary}
+here=$(cd "$(dirname "$0")" && pwd)
+
+readonly PORT=5000
+readonly STORE=/var/lib/nix-cache
+readonly SIGNING_KEY_NAME=smoke-test-1
+readonly WRITE_KEY=smoke-test-write-key
+# A NAR's file name is its hash in base 32; the server stores any name of
+# that shape.
+readonly NAR=0000000000000000000000000000000000000000000000000000.nar
+readonly BODY='smoke test'
+
+base=http://127.0.0.1:$PORT
+
+pass() { echo "ok    $*"; }
+fail() {
+  echo "FAIL  $*" >&2
+  sudo journalctl -u nova-cache.service --no-pager -n 40 >&2 || true
+  exit 1
+}
+
+# A store another user owns, as the host's is before this unit's first
+# deploy there.
+sudo mkdir -p "$STORE/nar"
+echo kept | sudo tee "$STORE/nar/pre-existing" > /dev/null
+sudo chown -R "$(id -un)" "$STORE"
+
+# An Ed25519 key as Nix writes one: the name, then the seed and the public
+# key in base 64.  Each is the last 32 bytes of its DER encoding.
+work=$(mktemp -d)
+trap 'rm -rf "$work"' EXIT
+openssl genpkey -algorithm ed25519 -out "$work/key.pem"
+{
+  printf '%s:' "$SIGNING_KEY_NAME"
+  {
+    openssl pkey -in "$work/key.pem" -outform DER | tail -c 32
+    openssl pkey -in "$work/key.pem" -pubout -outform DER | tail -c 32
+  } | base64 -w 0
+  echo
+} > "$work/secret-key"
+sudo install -D -m 0600 -o root -g root "$work/secret-key" /etc/nix-cache/secret-key
+printf 'CACHE_API_KEY=%s\n' "$WRITE_KEY" |
+  sudo sh -c 'umask 077; cat > /etc/nix-cache/env'
+
+sudo install -m 0755 "$server" /usr/local/bin/nova-cache-server
+sudo install -D -m 0644 "$here/nova-cache.sysusers" /etc/sysusers.d/nova-cache.conf
+sudo systemd-sysusers /etc/sysusers.d/nova-cache.conf
+sudo install -m 0644 "$here/nova-cache.service" /etc/systemd/system/nova-cache.service
+sudo systemctl daemon-reload
+sudo systemctl start nova-cache.service
+
+curl --fail --silent --show-error --output /dev/null \
+  --retry 10 --retry-connrefused --retry-delay 1 "$base/nix-cache-info" ||
+  fail "the server does not answer on loopback"
+[ "$(systemctl is-active nova-cache.service)" = active ] || fail "the service is not active"
+pass "the server runs under the unit and answers on loopback"
+
+# The landing page prints the public key of the signing key it loaded.
+curl --fail --silent --show-error "$base/" | grep -q -F "$SIGNING_KEY_NAME:" ||
+  fail "the landing page does not name the signing key, so the server did not read it"
+pass "the server read its signing key"
+
+code=$(printf '%s' "$BODY" | curl --silent --output /dev/null --write-out '%{http_code}' \
+  -X PUT -H "Authorization: Bearer $WRITE_KEY" --data-binary @- "$base/nar/$NAR")
+[ "$code" = 200 ] || fail "an authenticated write answered $code"
+[ "$(curl --fail --silent --show-error "$base/nar/$NAR")" = "$BODY" ] ||
+  fail "the written NAR does not read back"
+pass "an authenticated write is stored and read back"
+
+code=$(printf '%s' "$BODY" | curl --silent --output /dev/null --write-out '%{http_code}' \
+  -X PUT -H "Authorization: Bearer wrong" --data-binary @- "$base/nar/1${NAR#0}")
+[ "$code" = 401 ] || fail "a write with the wrong key answered $code"
+pass "a write with the wrong key is refused"
+
+for path in "$STORE" "$STORE/nar/pre-existing" "$STORE/nar/$NAR"; do
+  owner=$(sudo stat -c %U "$path")
+  [ "$owner" = nova-cache ] || fail "$path belongs to $owner"
+done
+pass "the store another user owned is the service's"
+
+address=$(hostname -I | awk '{ print $1 }')
+if curl --silent --max-time 3 --output /dev/null "http://$address:$PORT/nix-cache-info"; then
+  fail "the server answers on $address"
+fi
+pass "the server does not answer on $address"
+
+systemd-analyze security nova-cache.service --no-pager | tail -n 1